deploy: similarity feature + keyless providers (squashed for HF Spaces)

agents/__init__.py

@@ -11,3 +11,4 @@ from agents.claim_verifier_agent import ClaimVerifierAgent
 from agents.counter_factuality_agent import CounterFactualityAgent
 from agents.scoring_agent import ScoringAgent
 from agents.report_agent import ReportAgent
+from agents.similarity_agent import SimilarityAgent

agents/similarity_agent.py

@@ -0,0 +1,100 @@
+"""Similarity Agent (Wave 3 — parallel with claim/gap).
+
+Reads the paper text from Neo4j, runs the corpus-similarity pipeline (plan.md
+§8.1), and persists the `SimilarityReport` so the API can serve it verbatim.
+`critical = False`: a CORE outage or a timeout must degrade the report to
+`status="partial"`, never fail the whole job.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+from agents.base_agent import BaseAgent, AgentContext
+from similarity.config import settings
+from similarity.corpus.arxiv import ArxivProvider
+from similarity.corpus.core_api import CoreApiProvider
+from similarity.corpus.crossref import CrossrefProvider
+from similarity.corpus.internal import InternalCorpusProvider
+from similarity.corpus.openalex import OpenAlexProvider
+from similarity.pipeline import SimilarityPipeline
+
+
+def build_default_providers(mongo=None, *, doc_id: str, job_id: str | None = None):
+    """Instantiate the configured providers, respecting the SIM_ENABLE_* toggles."""
+    providers = []
+    if settings.enable_core:
+        providers.append(CoreApiProvider())
+    if settings.enable_arxiv:
+        providers.append(ArxivProvider())
+    if settings.enable_openalex:
+        providers.append(OpenAlexProvider())
+    if settings.enable_crossref:
+        providers.append(CrossrefProvider())
+    if settings.enable_internal and mongo is not None:
+        providers.append(
+            InternalCorpusProvider(mongo, doc_id=doc_id, job_id=job_id)
+        )
+    return providers
+
+
+class SimilarityAgent(BaseAgent):
+    name = "similarity"
+    wave = 3
+    critical = False
+
+    async def execute(self, ctx: AgentContext) -> Dict[str, Any]:
+        rows = ctx.neo4j.run(
+            "MATCH (d:Document {doc_id: $doc_id})-[:HAS_SECTION]->(:Section)-[:HAS_PARAGRAPH]->(p:Paragraph) "
+            "RETURN p.text AS text ORDER BY p.position",
+            doc_id=ctx.doc_id,
+        )
+        if not rows:
+            rows = ctx.neo4j.run(
+                "MATCH (d:Document {doc_id: $doc_id})-[:HAS_SECTION]->(s:Section) "
+                "RETURN s.text AS text ORDER BY s.position",
+                doc_id=ctx.doc_id,
+            )
+        text = " ".join(r["text"] for r in rows if r.get("text"))
+        if len(text.split()) < settings.min_document_words:
+            self.logger.info(f"[{ctx.job_id}] Text too short for similarity report")
+            return {"status": "unavailable", "overall_percent": 0}
+
+        providers = build_default_providers(ctx.mongo, doc_id=ctx.doc_id, job_id=ctx.job_id)
+        pipeline = SimilarityPipeline(
+            providers=providers,
+            doc_title=ctx.extra.get("title", "") if ctx.extra else "",
+            embedder=ctx.embedder if settings.enable_paraphrase else None,
+        )
+        report = await pipeline.run(text, doc_id=ctx.doc_id)
+        payload = report.model_dump(mode="json")
+
+        # Persist summary nodes so the report survives a restart (plan.md §8.1).
+        try:
+            for src in report.sources[:20]:
+                ctx.neo4j.run_write(
+                    """
+                    MATCH (d:Document {doc_id: $doc_id})
+                    MERGE (sim:SimilaritySource {source_index: $idx, doc_id: $doc_id})
+                    SET sim.title = $title, sim.url = $url, sim.bucket = $bucket,
+                        sim.matched_words = $words, sim.percent = $percent,
+                        sim.provider = $provider
+                    MERGE (d)-[:HAS_SIMILARITY_SOURCE]->(sim)
+                    """,
+                    doc_id=ctx.doc_id,
+                    idx=src.source_index,
+                    title=src.title,
+                    url=src.url,
+                    bucket=src.bucket.value,
+                    words=src.matched_words,
+                    percent=src.percent,
+                    provider=src.provider,
+                )
+        except Exception as e:
+            self.logger.warning(f"[{ctx.job_id}] Could not persist similarity nodes: {e}")
+
+        self.logger.info(
+            f"[{ctx.job_id}] similarity | status={report.status} "
+            f"overall={report.overall_percent}%"
+        )
+        return payload

backend/main.py

@@ -29,6 +29,7 @@ from backend.routers import (
     health,
     jobs,
     reports,
+    similarity,
     text_detection,
     websocket,
 )
@@ -125,6 +126,7 @@ def create_app() -> FastAPI:
     app.include_router(groq.router)
     app.include_router(jobs.router)
     app.include_router(reports.router)
+    app.include_router(similarity.router)
     app.include_router(text_detection.router)
     app.include_router(websocket.router)
 

backend/routers/jobs.py

@@ -121,4 +121,8 @@ async def delete_job(
 
     db = mongo._get_db()
     await db.jobs.delete_one({"job_id": job_id})
+
+    # plan.md 16 risk 6: a deleted job must take its corpus rows with it.
+    from similarity.index_writer import purge_corpus_for_job
+    await purge_corpus_for_job(mongo, job_id=job_id, doc_id=job.get("doc_id"))
     return {"deleted": job_id}

backend/routers/reports.py

@@ -16,6 +16,7 @@ from backend.schemas.reports import (
     GraphSubgraphResponse,
     GraphNodeResponse,
 )
+from similarity.schema import SimilarityReport
 from services.mongo_service import MongoService
 from services.neo4j_service import Neo4jService
 from services.email_service import EmailService
@@ -247,6 +248,33 @@ async def get_result_json(
     )
 
 
+@router.get(
+    "/{job_id}/similarity",
+    response_model=SimilarityReport,
+    summary="Get the corpus-similarity report for a job",
+)
+async def get_similarity_report(
+    job_id: str,
+    mongo: MongoService = Depends(get_mongo),
+):
+    """Serve the stored `SimilarityReport` (written by the orchestrator)."""
+    job = await mongo.get_job(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.")
+
+    result = await mongo.get_result(job_id)
+    report = (result or {}).get("similarity")
+    if not report:
+        raise HTTPException(
+            status_code=202,
+            detail=(
+                f"Similarity report not ready yet. Job status: "
+                f"{job.get('status', 'unknown')}"
+            ),
+        )
+    return report
+
+
 @router.get(
     "/{job_id}/graph",
     response_model=GraphSubgraphResponse,

backend/routers/similarity.py

@@ -0,0 +1,164 @@
+"""Corpus-similarity endpoints (plan.md §9).
+
+    GET  /similarity/status          -> 200 SimilarityStatus
+    POST /similarity/{job_id}/recheck -> 202 {job_id, status} | 409 | 404
+
+The report itself lives at `GET /reports/{job_id}/similarity` in reports.py
+beside /json, /pdf and /graph.
+
+`recheck` re-runs the pipeline against the current corpus using the text the
+parser already stored in Neo4j - no PDF re-parse, no full re-analysis. It runs
+as a BackgroundTask (same as POST /analyze) and is guarded by an in-process
+set keyed by job_id: four clicks on the button must not launch four concurrent
+24-query provider storms. The guard is process-local, which matches how the
+whole app is deployed today (single uvicorn worker).
+"""
+from __future__ import annotations
+
+from typing import Callable
+
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
+
+from backend.dependencies import get_embedder, get_mongo, get_neo4j
+from backend.schemas.similarity import RecheckAccepted, SimilarityStatus
+from services.mongo_service import MongoService
+from services.neo4j_service import Neo4jService
+from services.vector_store_service import VectorStoreService
+from similarity.config import settings as sim_settings
+from utils.logger import get_logger
+
+logger = get_logger("backend.routers.similarity")
+
+router = APIRouter(prefix="/api/similarity", tags=["Similarity"])
+
+# In-process guard: job_ids whose recheck is currently running.
+_recheck_in_flight: set[str] = set()
+
+
+def get_recheck_in_flight() -> set[str]:
+    """Expose the in-flight guard (also used by tests to seed a 409)."""
+    return _recheck_in_flight
+
+
+def _enabled_providers() -> list[str]:
+    """Provider names currently switched on, in pipeline order."""
+    toggles = (
+        (sim_settings.enable_core, "core"),
+        (sim_settings.enable_arxiv, "arxiv"),
+        (sim_settings.enable_openalex, "openalex"),
+        (sim_settings.enable_crossref, "crossref"),
+        (sim_settings.enable_internal, "internal"),
+    )
+    return [name for on, name in toggles if on]
+
+
+async def _run_recheck(
+    job_id: str,
+    mongo: MongoService,
+    neo4j: Neo4jService,
+    embedder: VectorStoreService,
+) -> None:
+    """Re-run the similarity pipeline for a completed job's stored text."""
+    try:
+        job = await mongo.get_job(job_id)
+        doc_id = (job or {}).get("doc_id")
+        if not doc_id:
+            logger.warning(f"[{job_id}] recheck skipped: no doc_id")
+            return
+
+        rows = neo4j.run(
+            "MATCH (d:Document {doc_id: $doc_id})-[:HAS_SECTION]->(:Section)-[:HAS_PARAGRAPH]->(p:Paragraph) "
+            "RETURN p.text AS text ORDER BY p.position",
+            doc_id=doc_id,
+        )
+        if not rows:
+            rows = neo4j.run(
+                "MATCH (d:Document {doc_id: $doc_id})-[:HAS_SECTION]->(s:Section) "
+                "RETURN s.text AS text ORDER BY s.position",
+                doc_id=doc_id,
+            )
+        text = " ".join(r["text"] for r in rows if r.get("text"))
+
+        from agents.similarity_agent import build_default_providers
+        from similarity.pipeline import SimilarityPipeline
+
+        providers = build_default_providers(mongo, doc_id=doc_id, job_id=job_id)
+        pipeline = SimilarityPipeline(
+            providers=providers,
+            embedder=embedder if sim_settings.enable_paraphrase else None,
+        )
+        report = await pipeline.run(text, doc_id=doc_id)
+        payload = report.model_dump(mode="json")
+
+        # save_result merges with $set, so this refreshes just the similarity
+        # key without clobbering the rest of the analysis.
+        await mongo.save_result(job_id, {"similarity": payload})
+        logger.info(
+            f"[{job_id}] similarity recheck done | status={report.status} "
+            f"overall={report.overall_percent}%"
+        )
+    except Exception as e:
+        logger.error(f"[{job_id}] similarity recheck failed: {e}")
+    finally:
+        _recheck_in_flight.discard(job_id)
+
+
+def get_recheck_runner():
+    """Dependency returning the background recheck runner (overridable)."""
+    return _run_recheck
+
+
+@router.get(
+    "/status",
+    response_model=SimilarityStatus,
+    summary="Which similarity providers are configured, corpus size, thresholds",
+)
+async def similarity_status(
+    mongo: MongoService = Depends(get_mongo),
+) -> SimilarityStatus:
+    enabled = _enabled_providers()
+    corpus_size = 0
+    try:
+        db = mongo._get_db()
+        corpus_size = await db.corpus_fingerprints.count_documents({})
+    except Exception:
+        corpus_size = 0  # index not yet created -> nothing indexed
+    return SimilarityStatus(
+        enabled_providers=enabled,
+        any_enabled=bool(enabled),
+        corpus_size=corpus_size,
+        min_document_words=sim_settings.min_document_words,
+        min_match_words=sim_settings.min_match_words,
+        budget_seconds=sim_settings.budget_seconds,
+        paraphrase_enabled=sim_settings.enable_paraphrase,
+        paraphrase_threshold=sim_settings.paraphrase_threshold,
+    )
+
+
+@router.post(
+    "/{job_id}/recheck",
+    response_model=RecheckAccepted,
+    status_code=202,
+    summary="Re-run the corpus-similarity report for a completed job",
+)
+async def recheck_similarity(
+    job_id: str,
+    background_tasks: BackgroundTasks,
+    mongo: MongoService = Depends(get_mongo),
+    neo4j: Neo4jService = Depends(get_neo4j),
+    embedder: VectorStoreService = Depends(get_embedder),
+    runner: Callable = Depends(get_recheck_runner),
+) -> RecheckAccepted:
+    job = await mongo.get_job(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.")
+
+    if job_id in _recheck_in_flight:
+        raise HTTPException(
+            status_code=409,
+            detail=f"Recheck for job '{job_id}' is already in flight.",
+        )
+
+    _recheck_in_flight.add(job_id)
+    background_tasks.add_task(runner, job_id, mongo, neo4j, embedder)
+    return RecheckAccepted(job_id=job_id)

backend/schemas/similarity.py

@@ -0,0 +1,34 @@
+"""
+Pydantic schemas for the corpus-similarity endpoints (plan.md §9).
+
+The report body is `similarity.schema.SimilarityReport`, reused directly as
+the response model the way text_detection reuses `EnsembleResult` - a parallel
+copy would drift from the package it describes. Only the operator-facing status
+view and the recheck acknowledgement live here.
+"""
+
+from pydantic import BaseModel, Field
+
+
+class SimilarityStatus(BaseModel):
+    """Which providers are live and what thresholds the pipeline will use.
+
+    Directly modelled on `GET /text-detection/status`: it is how an operator
+    distinguishes "nothing configured" from "configured but failing".
+    """
+
+    enabled_providers: list[str] = Field(default_factory=list)
+    any_enabled: bool
+    corpus_size: int = 0
+    min_document_words: int
+    min_match_words: int
+    budget_seconds: int
+    paraphrase_enabled: bool
+    paraphrase_threshold: float
+
+
+class RecheckAccepted(BaseModel):
+    """Acknowledgement that a recheck has been queued (plan.md §9)."""
+
+    job_id: str
+    status: str = "rechecking"

backend/tests/conftest.py

@@ -78,6 +78,10 @@ def mock_mongo():
     mock_db.command = AsyncMock(return_value={"ok": 1})
     mock_db.jobs = MagicMock()
     mock_db.jobs.delete_one = AsyncMock()
+    mock_db.corpus_fingerprints = MagicMock()
+    mock_db.corpus_fingerprints.delete_many = AsyncMock()
+    mock_db.corpus_texts = MagicMock()
+    mock_db.corpus_texts.delete_many = AsyncMock()
     m._get_db = MagicMock(return_value=mock_db)
     return m
 

backend/tests/test_jobs.py

@@ -62,6 +62,14 @@ def test_delete_job(client):
     assert resp.json()["deleted"] == "job_test123"
 
 
+def test_delete_job_purges_corpus_rows(client, mock_mongo):
+    db = mock_mongo._get_db()
+    resp = client.delete("/api/jobs/job_test123")
+    assert resp.status_code == 200
+    db.corpus_fingerprints.delete_many.assert_awaited_once()
+    db.corpus_texts.delete_many.assert_awaited_once()
+
+
 def test_delete_job_not_found(client, mock_mongo):
     mock_mongo.get_job = AsyncMock(return_value=None)
     resp = client.delete("/api/jobs/nonexistent")

backend/tests/test_similarity.py

@@ -0,0 +1,146 @@
+"""Tests for the corpus-similarity endpoints (plan.md §9, prompt Phase 4).
+
+    GET  /reports/{job_id}/similarity   -> 200 SimilarityReport | 202 | 404
+    POST /similarity/{job_id}/recheck   -> 202 {job_id, status} | 409 in flight | 404
+    GET  /similarity/status             -> 200 SimilarityStatus
+
+The recheck background task must not hit real providers in tests: the runner
+is overridden through a dependency override, so only the 202/409/404 guard
+behaviour is exercised here.
+"""
+import pytest
+from unittest.mock import AsyncMock
+
+from backend.routers.similarity import get_recheck_in_flight, get_recheck_runner
+from similarity.schema import Bucket, Coverage, MatchSpan, SimilarityReport, SourceMatch
+
+
+def _report_json() -> dict:
+    report = SimilarityReport(
+        overall_percent=42,
+        bucket_percents={Bucket.OPEN_ACCESS: 30, Bucket.PUBLICATION: 12},
+        sources=[
+            SourceMatch(
+                source_index=1,
+                bucket=Bucket.OPEN_ACCESS,
+                title="A real source",
+                url="https://example.org/paper",
+                doi="10.1000/x",
+                authors=["Alice", "Bob"],
+                year=2023,
+                provider="CORE",
+                display_label="https://example.org/paper",
+                matched_words=120,
+                percent=30,
+                spans=[
+                    MatchSpan(
+                        doc_start_word=0, doc_end_word=8, doc_char_start=0, doc_char_end=40,
+                        word_count=8, excerpt="some shared text", source_excerpt="some shared text",
+                        source_index=1,
+                    )
+                ],
+            )
+        ],
+        coverage=Coverage(
+            total_words_compared=400,
+            words_excluded=10,
+            exclusions_applied=["bibliography"],
+            providers_queried=["CORE", "arXiv"],
+            phrases_queried=24,
+            candidates_retrieved=42,
+            candidates_verified=8,
+            internal_corpus_size=0,
+            min_match_words=8,
+        ),
+        status="complete",
+        notes=[],
+    )
+    return report.model_dump(mode="json")
+
+
+@pytest.fixture(autouse=True)
+def _clear_recheck_guard():
+    get_recheck_in_flight().clear()
+    yield
+    get_recheck_in_flight().clear()
+
+
+@pytest.fixture
+def stub_recheck_runner(client, mock_mongo, mock_neo4j):
+    """Override the background recheck runner so no provider is hit."""
+    calls = []
+
+    async def fake_runner(job_id, mongo, neo4j, embedder):
+        calls.append(job_id)
+
+    client.app.dependency_overrides[get_recheck_runner] = lambda: fake_runner
+    yield calls
+    client.app.dependency_overrides.pop(get_recheck_runner, None)
+
+
+# ── GET /reports/{job_id}/similarity ─────────────────────────────────────────
+
+
+def test_similarity_report_served(client, mock_mongo):
+    mock_mongo.get_result = AsyncMock(return_value={"similarity": _report_json()})
+    resp = client.get("/api/reports/job_test123/similarity")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["overall_percent"] == 42
+    assert body["status"] == "complete"
+    assert body["coverage"]["phrases_queried"] == 24
+    assert body["sources"][0]["provider"] == "CORE"
+
+
+def test_similarity_report_202_when_job_pending(client, mock_mongo):
+    mock_mongo.get_result = AsyncMock(return_value=None)
+    mock_mongo.get_job = AsyncMock(return_value={"job_id": "job_test123", "status": "running"})
+    resp = client.get("/api/reports/job_test123/similarity")
+    assert resp.status_code == 202
+
+
+def test_similarity_report_404_unknown_job(client, mock_mongo):
+    mock_mongo.get_job = AsyncMock(return_value=None)
+    resp = client.get("/api/reports/nonexistent/similarity")
+    assert resp.status_code == 404
+
+
+# ── POST /similarity/{job_id}/recheck ────────────────────────────────────────
+
+
+def test_recheck_accepted(client, stub_recheck_runner):
+    resp = client.post("/api/similarity/job_test123/recheck")
+    assert resp.status_code == 202
+    body = resp.json()
+    assert body["job_id"] == "job_test123"
+    assert body["status"] == "rechecking"
+    assert "job_test123" in stub_recheck_runner
+
+
+def test_recheck_409_when_already_in_flight(client, stub_recheck_runner, mock_mongo):
+    get_recheck_in_flight().add("job_test123")
+    resp = client.post("/api/similarity/job_test123/recheck")
+    assert resp.status_code == 409
+    assert "job_test123" not in stub_recheck_runner  # no second storm launched
+
+
+def test_recheck_404_unknown_job(client, mock_mongo, stub_recheck_runner):
+    mock_mongo.get_job = AsyncMock(return_value=None)
+    resp = client.post("/api/similarity/nonexistent/recheck")
+    assert resp.status_code == 404
+
+
+# ── GET /similarity/status ──────────────────���────────────────────────────────
+
+
+def test_status_shape(client):
+    resp = client.get("/api/similarity/status")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert "enabled_providers" in body
+    assert "any_enabled" in body
+    assert "corpus_size" in body
+    assert "min_document_words" in body
+    assert "min_match_words" in body
+    assert "budget_seconds" in body
+    assert "paraphrase_enabled" in body

configs/pipeline_config.yaml

@@ -35,6 +35,10 @@ agents:
     enabled: true
     wave: 4
     depends_on: [argumentation, citation_gap]
+  similarity:
+    enabled: true
+    wave: 3
+    depends_on: [parser, keyword]
   report:
     enabled: true
     wave: 5

opencode-prompt.md

@@ -0,0 +1,357 @@
+# OpenCode Implementation Prompt — Corpus Similarity Report
+
+> Paste everything below the line into OpenCode as the opening prompt.
+> Run it from the repo root (`IIT-Patna/`).
+
+---
+
+## Your task
+
+> **State as of merge `7b89d44`:** Phases 0–4 are implemented, merged with the remote,
+> and pushed. **Start at Phase 5.** All API routes now live under `/api/*`; read the
+> post-merge amendments box at the top of `plan.md` before doing anything.
+
+Implement the Corpus Similarity Report feature specified in `plan.md` in this
+repository. Work phase by phase, test-first, committing after each phase.
+
+`plan.md` is the **specification and the source of truth**. This prompt tells you how to
+execute it. Where they conflict, `plan.md` wins on *what* and this prompt wins on *how*.
+
+**Read these before writing any code, in this order:**
+
+1. `plan.md` — the full spec. Read all 18 sections. Do not skim §4 (the scoring math) or
+   §7.4 (the privacy boundary); those are the two places a plausible-looking
+   implementation is most likely to be wrong.
+2. `text_detection/` — the package you are copying the shape of. Read `config.py`,
+   `schema.py`, `pipeline.py`, `detectors/base.py`. Note: env-prefixed settings,
+   `extra="ignore"`, everything ships disabled, pipeline degrades instead of raising.
+3. `services/paper_recommender.py` — the provider clients, rate-limit semaphore
+   (`_PROVIDER_SEM`), `STOPWORDS` / `FILLER_TOKENS`, `source_for()`, `DOI_PUBLISHER`.
+   You will import from this file. Do not copy from it.
+4. `services/pdf_highlight_mapper.py` — the existing PDF span→coordinate mapper you will
+   extend in Phase 5.
+5. `orchestrators/custom_orchestrator.py` and `workflows/hybrid_workflow.py` — how agents
+   are registered, how waves execute, what `critical=False` actually does.
+6. `agents/base_agent.py` — the `BaseAgent` / `AgentContext` / `AgentResult` contract.
+7. `backend/backend.md` — the API conventions you must match.
+
+---
+
+## Stack facts you need
+
+- Python 3.13, FastAPI, `pydantic` v2, `pydantic-settings`, `motor` (async Mongo),
+  `neo4j` (sync driver), `httpx`, `pytest` + `pytest-asyncio`.
+- Frontend: React 18 + Vite, plain JSX (no TypeScript), one CSS file per page/component.
+- Run the backend: `uvicorn backend.main:app --reload --port 8000`
+- Run tests: `python -m pytest tests/similarity/ -v` and
+  `python -m pytest backend/tests/ -v`
+- Build the frontend: `cd frontend-app && npm run build`
+- Windows dev machine. Use `pathlib`, never string concatenation, for paths.
+
+---
+
+## Non-negotiable constraints
+
+Violating any of these means the work gets rejected regardless of whether it runs.
+
+1. **No new runtime dependencies.** `requirements.txt` must not grow. Winnowing is
+   stdlib. If you believe you need a package, stop and ask — you almost certainly do not.
+2. **TDD, genuinely.** Write the failing test, run it, watch it fail for the right
+   reason, then implement. A test written after the code is not a test, it is a
+   description. Minimum 80% coverage on the `similarity/` package.
+3. **File size:** 200–400 lines typical, 800 hard maximum. Two files are already at the
+   edge — see "Traps" below.
+4. **No mutation.** Build new objects; do not modify inputs in place. The scoring
+   functions in `similarity/aggregate.py` must be pure.
+5. **Never swallow an error.** A provider that 429s goes into
+   `coverage.providers_failed`. A span that cannot be located gets logged and omitted,
+   never guessed. There is no bare `except: pass` anywhere in this feature.
+6. **No LLM in the scoring path.** The percentage is computed arithmetically from
+   verified spans. `LLMService` must not be imported anywhere under `similarity/`.
+7. **The privacy boundary in `plan.md` §7.4 is a hard requirement.** An internal-corpus
+   match must never carry the other document's title, URL, authors, owner, or text. This
+   is enforced at construction in `corpus/internal.py`, not filtered downstream. There is
+   a test for it; do not weaken the test.
+8. **Never fabricate a source.** Every `SourceMatch.url` must be a real, resolvable URL
+   returned by a provider. A candidate without a URL is dropped, exactly as
+   `_score_candidates()` in `paper_recommender.py` already does.
+
+---
+
+## Patterns to follow
+
+| Need | Copy from | Not from |
+|---|---|---|
+| Package layout, settings, degradation | `text_detection/` | anywhere else |
+| Env-prefixed settings class | `text_detection/config.py` (`TEXTDET_` → yours is `SIM_`) | `backend/config.py` |
+| Async provider client + retry + throttle | `services/paper_recommender.py` (`@async_retry`, `_PROVIDER_SEM`) | write your own |
+| Agent class shape | `agents/citation_gap_agent.py` (176 lines, `critical=False`, wave 3) | `agents/report_agent.py` (775 lines) |
+| Router shape | `backend/routers/text_detection.py` | `backend/routers/reports.py` |
+| Test fixtures / dependency overrides | `backend/tests/conftest.py` | write your own |
+| Status endpoint | `GET /text-detection/status` | invent a new shape |
+
+---
+
+## Execution order
+
+Do these in order. **Commit at the end of each phase**, with a conventional-commit
+message (`feat:`, `test:`, `refactor:`). Do not start a phase until the previous phase's
+exit criteria pass.
+
+### Phase 0 — Scoring core (no I/O)
+
+Create `similarity/{__init__,config,schema,normalize,exclusions,fingerprint,selector,matcher,aggregate}.py`.
+
+Everything here is a pure function over strings and lists. No network, no database, no
+async. This is deliberate: it is the part that must be provably correct, so it must be
+testable in milliseconds.
+
+Implement in this order, each with its tests first:
+
+1. `schema.py` — the Pydantic models exactly as written in `plan.md` §6.1. Copy the field
+   names verbatim; later phases and the frontend depend on them.
+2. `config.py` — `SimilaritySettings(BaseSettings)` with `env_prefix="SIM_"`,
+   `extra="ignore"`, every field and default from `plan.md` §12. Module-level
+   `settings = SimilaritySettings()`, matching `text_detection/config.py`.
+3. `normalize.py` — `normalize(text) -> NormalizedDoc` carrying the word list plus an
+   offset map back into the original string. **The offset map is what makes highlighting
+   land on the right pixels.** Test it by round-tripping every word.
+4. `exclusions.py` — find bibliography, quoted text, title block, equations, captions;
+   return index ranges to exclude. Reuse `utils.pdf.split_into_sections` for the
+   bibliography — `ParserAgent` already isolates it.
+5. `fingerprint.py` — k-gram hashing + winnowing. `k = settings.kgram_size` (5 words),
+   `w = settings.window_size` (4). The correctness test is a **property test**: generate
+   random text pairs sharing a run of ≥ `w+k-1` words, assert the fingerprint sets always
+   intersect. That guarantee is the foundation of every recall claim in the spec.
+6. `selector.py` — pick the query phrases per `plan.md` §3 ("Choosing which phrases").
+   Import `STOPWORDS` and `FILLER_TOKENS` from `services.paper_recommender`.
+7. `matcher.py` — fingerprint seeds → verified spans with exact word and char offsets.
+   Overlapping seeds merge into one span.
+8. `aggregate.py` — the coverage bitmap and all four percentage calculations from
+   `plan.md` §4. This is the most important file in the feature.
+
+**Exit criteria:**
+- `python -m pytest tests/similarity/ -v` — all green, including the winnowing property
+  test.
+- A document scored against itself returns exactly `100`.
+- A document scored against unrelated text returns exactly `0`.
+- `test_aggregate.py` covers every case listed in `plan.md` §13, especially: one span
+  claimed by three sources counts **once**.
+
+### Phase 1 — Internal corpus
+
+Create `similarity/corpus/{__init__,base}.py`, `similarity/corpus/internal.py`, and
+`similarity/index_writer.py`.
+
+- `base.py` — the `CorpusProvider` protocol and `Candidate` dataclass from `plan.md` §7.
+- Mongo collections `corpus_fingerprints` and `corpus_texts` per `plan.md` §6.2. Create
+  the multikey index on `fingerprints` at startup — add it beside the existing index
+  creation in the orchestrator, not on every query.
+- `index_writer.py` — gated on `settings.index_uploads`, which defaults to `False`.
+- **Also in this phase:** extend `DELETE /jobs/{job_id}` in `backend/routers/jobs.py` to
+  purge that document's `corpus_fingerprints` and `corpus_texts` rows. `plan.md` §16
+  risk 6 requires this before the corpus can ever be enabled; build it now while the
+  write path is fresh, not later.
+- Self-match guard: exclude by `doc_id`, and on recheck by `job_id`.
+
+**Exit criteria:**
+- Index document A, then score A against the corpus ⇒ ~100%.
+- `tests/similarity/test_privacy.py` green: the returned `SourceMatch` for an internal hit
+  carries no title, URL, author, or text from the other document — only the neutral label
+  `CitationEdge Corpus · submitted YYYY-MM-DD`.
+- Deleting a job removes its corpus rows.
+
+### Phase 2 — External providers
+
+Create `similarity/corpus/{core_api,arxiv,openalex,crossref}.py`.
+
+- **CORE is the workhorse.** `GET https://api.core.ac.uk/v3/search/works` with
+  `q=fullText:"<phrase>"`. The response can carry `fullText`, so retrieval and text fetch
+  are one round trip. Send `Authorization: Bearer <SIM_CORE_API_KEY>` when the key is
+  set; log a startup warning when it is not (unregistered is ~5 requests / 10 seconds and
+  will rate-limit you immediately).
+- arXiv / OpenAlex / Crossref: **import** `_search_arxiv`, `_search_openalex`,
+  `_search_crossref`, `source_for`, and `DOI_PUBLISHER` from
+  `services.paper_recommender`. If clean importing requires lifting them into a shared
+  `services/academic_search.py` that both modules import, do that refactor — it is
+  in-scope. Do not duplicate the functions.
+- Every provider: shared `asyncio.Semaphore(2)`, stagger between calls, `@async_retry`
+  from `utils.retry`, and on persistent failure append to `coverage.providers_failed` and
+  return `[]`.
+
+**Tests: no network.** Use `httpx.MockTransport` with fixtures covering happy path, 429,
+5xx, malformed JSON, empty results, and a candidate with no resolvable URL.
+
+**Exit criteria:** provider tests green; one manual run against a paper with a known arXiv
+preprint locates it.
+
+### Phase 3 — Pipeline and agent
+
+Create `similarity/{pipeline,semantic}.py` and `agents/similarity_agent.py`.
+
+- `pipeline.py` — orchestrates normalize → exclude → select → retrieve → verify → match →
+  aggregate → report. Wrap the whole retrieval stage in
+  `asyncio.wait_for(..., timeout=settings.budget_seconds)`; on timeout return what is
+  verified so far with `status="partial"` and `coverage.budget_exhausted=True`.
+- **Implement the preprint self-match guard here** (`plan.md` §16 risk 2) — drop
+  candidates whose normalized title overlaps the uploaded title by ≥ 0.9 tokens, or whose
+  DOI matches, or whose authors overlap ≥ 50% while similarity > 60%. Record the drop in
+  `coverage.notes`. This needs its own test fixture. Skipping it produces a ~100% score on
+  every already-published paper.
+- `semantic.py` — the paraphrase pass using the existing `VectorStoreService`. Assert in
+  a test that it **cannot** change `overall_percent`.
+- `agents/similarity_agent.py` — `name = "similarity"`, `wave = 3`, `critical = False`.
+  Keep it under 100 lines: read paragraphs from Neo4j (reuse the query the orchestrator
+  already runs for AI-text detection), call the pipeline, write summary nodes, return the
+  dict.
+- Wire into `orchestrators/custom_orchestrator.py`: add to `get_agents()`, read the
+  result off the `AgentResult` list, add `similarity` to `final_result`, `upsert_job`,
+  and `save_result`. `save_result` merges with `$set`, so adding a key is non-breaking.
+- Add the `similarity` entry to `configs/pipeline_config.yaml` (wave 3,
+  `depends_on: [parser, keyword]`).
+
+**Exit criteria:** a full analysis job writes `results.similarity`; simulating a CORE
+outage yields `status="partial"` **and a completed job** — not a failed one.
+
+### Phase 4 — API
+
+- `GET /reports/{job_id}/similarity` in `backend/routers/reports.py`.
+- New `backend/routers/similarity.py`: `POST /{job_id}/recheck` (202, `BackgroundTask`,
+  reuses stored normalized text — **no PDF re-parse**) and `GET /status`.
+- New `backend/schemas/similarity.py` for the request/status models. Reuse
+  `similarity.schema.SimilarityReport` directly as the response model, the way
+  `text_detection` reuses `EnsembleResult` — do not mirror it.
+- Register the router in `backend/main.py`.
+- Recheck needs a `409` guard: an in-process set keyed by `job_id`, so four clicks do not
+  launch four concurrent 24-query provider storms.
+
+**Exit criteria:** `backend/tests/test_similarity.py` green (report, 202 pending, 404
+unknown, recheck accepted, recheck 409, status shape); endpoints visible at `/docs`.
+
+### Phase 5 — Highlighting
+
+Modify `services/pdf_highlight_mapper.py`.
+
+- Generalize the hardcoded span typing. Today `_match_span()` does
+  `"type": "ai" if span.get("verdict") == "AI_GENERATED" else "human"` and `run()`
+  filters to `verdict in ("AI_GENERATED", "REAL")`. Add an optional `kind` on the input
+  span that passes through when present, plus an optional `meta` dict carrying
+  `source_index` for per-source colouring. **Default behaviour must not change** — the
+  existing AI-detection tests must pass untouched.
+- Call the mapper **once**, not twice. It rasterizes into `pages_dir` and returns the full
+  `pages` array; two callers would race on the PNGs and produce two `page_highlights`
+  structures the frontend cannot merge. **The sole call site is
+  `agents/report_agent.py:327`** — not the orchestrator. `report` is wave 5 and
+  `similarity` is wave 3, so ReportAgent already has both span sets. Do not add a second
+  call site; extend the existing one with the similarity spans, each tagged with `kind`.
+
+**Exit criteria:** similarity spans render as boxes on real page images; every existing
+`pdf_highlight_mapper` test still passes with no edits.
+
+### Phase 6 — Frontend
+
+**Refactor first.** `frontend-app/src/pages/ReportPage.jsx` is **826 lines — already over
+the 800-line cap**. Before adding anything:
+
+- Move `buildReport()` and its helpers → `pages/reportAdapter.js`
+- Move the `DEMO` constant → `pages/reportDemo.js`
+- Verify the page still renders identically before continuing.
+
+Then create:
+
+- `components/SimilarityIndex.jsx` + `.css` — the header block: one large percentage,
+  three bucket figures. Brand rules from `DESIGN.md`: obsidian background, glass card,
+  Instrument Serif for the number, DM Mono for the buckets. **Not red.** Red is the error
+  colour in this brand; colouring a similarity score red makes an accusation the data
+  does not support.
+- `components/SimilaritySources.jsx` — the numbered, colour-chipped source list. Internal
+  corpus rows render as plain text, never links.
+- `components/CoverageNote.jsx` — the disclosure from `plan.md` §10.2, always visible,
+  never behind a toggle.
+- `api.js` — `getSimilarity(jobId)`, `recheckSimilarity(jobId)`.
+
+Wire into `ReportPage.jsx` in ~8 lines. Pass `kind === 'similarity'` highlights to the
+existing page viewer.
+
+**Exit criteria:** page renders the block; `ReportPage.jsx` back under 800 lines;
+`status: "unavailable"` renders **no percentage at all**; `npm run build` clean.
+
+### Phase 7 — PDF report section
+
+- Create `agents/report_sections/similarity_section.py` exporting
+  `build_similarity_section(report: dict) -> list[Flowable]`.
+- `agents/report_agent.py` is 775 lines — it gets a two-line import-and-append, nothing
+  more.
+- Place the section after AI-text detection, before citation gaps.
+- The coverage note must appear **verbatim** in the PDF. The PDF is the artefact that gets
+  emailed and attached to decisions; the disclosure has to travel with it.
+- When similarity is `unavailable`, render a one-line "not available for this document"
+  note rather than omitting the section, so a reader can tell the check ran.
+
+### Phase 8 — Evaluation
+
+- `scripts/eval_similarity.py`, modelled on `scripts/eval_text_detection.py`.
+- Build synthetic documents with known copy rates (0 / 5 / 10 / 25 / 50%) at run lengths
+  10 / 50 / 200 words, from CORE open-access papers.
+- Tune `SIM_MIN_MATCH_WORDS` and the boilerplate stoplist against the 0% set until the
+  false-positive rate is **< 1%**, *then* measure mean absolute error. Precision first: an
+  inflated score on an honest paper is far worse than a missed match on a dishonest one.
+- Write `similarity_eval_report.md` with the measured numbers. Do not write target numbers
+  as if they were results.
+- Update `backend/backend.md` (new endpoints) and `README.md` (feature list).
+
+---
+
+## Traps — read before you start
+
+These are real conditions in this repo that will bite you.
+
+1. **`ReportPage.jsx` is 869 lines, already over the cap.** Adding to it without the
+   Phase 6 extraction makes a bad file worse.
+2. **`report_agent.py` is 979 lines.** Well past the cap. The Phase 7 extraction is a
+   precondition for Phase 5 touching that file, not a later cleanup.
+3. **`PdfHighlightMapper` rasterizes pages as a side effect.** Two callers race. One call
+   with a combined span list.
+4. **`save_result()` merges with `$set`.** Adding `similarity` is safe; replacing the
+   document is not.
+5. **`critical = True` aborts the pipeline.** Read `HybridWorkflow.execute` — a failed
+   critical agent breaks the wave loop. Similarity must be `critical = False` or a CORE
+   outage costs users their entire analysis.
+6. **A published paper matches its own preprint at ~100%.** The Phase 3 guard is not
+   optional polish; without it the feature is unusable on real papers.
+7. **CORE unauthenticated is ~5 requests / 10 seconds.** 24 phrase queries will 429
+   instantly. Throttle from the first line of code, not after you see the error.
+8. **Bucket percentages do not sum to the headline number** (`plan.md` §4.4). This is
+   correct and matches Turnitin. Do not "fix" it.
+9. **Scanned PDFs have no text layer** ⇒ no fingerprints ⇒ 0%, which reads as "clean" but
+   means "we could not look". `SIM_MIN_DOCUMENT_WORDS` must force `unavailable`.
+
+---
+
+## Stop and ask if
+
+- A phase's exit criteria cannot be met without changing the spec in `plan.md`.
+- You believe a new dependency is required.
+- The privacy boundary in §7.4 seems to conflict with a UI requirement.
+- CORE's API shape differs from what `plan.md` §7.1 describes (the spec was written from
+  their docs; verify against the live API in Phase 2 and report any mismatch rather than
+  silently adapting).
+- Measured false-positive rate in Phase 8 stays above 1% after calibration — that is a
+  product decision, not a tuning problem.
+
+Do **not** stop to ask whether to continue between phases. Finish the phase, commit,
+report the exit criteria you verified, and start the next one.
+
+---
+
+## Reporting
+
+After each phase, report:
+
+- Which exit criteria you verified, and the **actual command output** that proves it.
+- Any file that grew past 400 lines, and why.
+- Anything in `plan.md` that turned out to be wrong once you hit the real code.
+
+Do not report a phase complete on the basis of code that looks right. Run the tests and
+paste what they said. If something fails and you cannot fix it, say so plainly and move
+to what you can finish.

orchestrators/custom_orchestrator.py

@@ -21,6 +21,7 @@ from agents import (
     ArgumentationAgent,
     ScoringAgent,
     ReportAgent,
+    SimilarityAgent,
     AgentContext,
     AgentStatus,
 )
@@ -76,6 +77,7 @@ class CitationEdgeOrchestrator(BaseOrchestrator):
             ArgumentationAgent(),
             ScoringAgent(),
             ReportAgent(),
+            SimilarityAgent(),
         ]
 
     async def run(
@@ -105,6 +107,14 @@ class CitationEdgeOrchestrator(BaseOrchestrator):
         except Exception as e:
             logger.warning(f"Index creation skipped: {e}")
 
+        # Ensure the internal-corpus fingerprint index (plan.md 6.2). Created
+        # once at startup, not on the per-query hot path.
+        try:
+            from similarity.index_writer import ensure_corpus_indexes
+            await ensure_corpus_indexes(self.mongo)
+        except Exception as e:
+            logger.warning(f"Corpus index creation skipped: {e}")
+
         # Build shared context (passed to every agent)
         graph_memory = GraphMemory(
             neo4j=self.neo4j, embedder=self.embedder, vector_db=self.vector_db
@@ -138,6 +148,13 @@ class CitationEdgeOrchestrator(BaseOrchestrator):
                 visual_figures = r.data["figures"]
                 break
 
+        # Extract the corpus-similarity report from the SimilarityAgent result
+        similarity_report = None
+        for r in results:
+            if r.agent_name == "similarity" and r.data:
+                similarity_report = r.data
+                break
+
         # Build summary
         duration = round(time.monotonic() - start, 2)
         agent_summary = {
@@ -342,6 +359,7 @@ class CitationEdgeOrchestrator(BaseOrchestrator):
             "scores": scores,
             "report_path": f"reports/{job_id}.pdf",
             "ai_text_detection": ai_text_detection,
+                "similarity": similarity_report,
         }
 
         # Persist final status and all results
@@ -353,6 +371,7 @@ class CitationEdgeOrchestrator(BaseOrchestrator):
                 "agents": agent_summary,
                 "scores": scores,
                 "ai_text_detection": ai_text_detection,
+                "similarity": similarity_report,
             },
         )
 
@@ -378,6 +397,7 @@ class CitationEdgeOrchestrator(BaseOrchestrator):
                 "citation_relevance": citrel_summary,
                 "irrelevant_citations": irrelevant_citations,
                 "ai_text_detection": ai_text_detection,
+                "similarity": similarity_report,
             },
         )
 

plan.md

@@ -0,0 +1,918 @@
+# Implementation Plan — Corpus Similarity Report ("Originality Report")
+
+**Feature:** Turnitin-style similarity/originality reporting for uploaded papers.
+**Status:** Phases 0–4 implemented; merged with remote `report-ui-updated` (merge commit
+`506ad88`). Phases 5–8 outstanding.
+**Date:** 2026-08-02 · revised after the merge
+
+> ### Post-merge amendments (read before implementing Phases 5–8)
+>
+> The remote branch landed 15 commits while Phases 0–4 were being built. Four of them
+> change assumptions in this document. The sections below have been corrected in place;
+> this box is the summary.
+>
+> 1. **Every API route now lives under `/api`** (`d58fb43`). Each router carries its own
+>    prefix and the SPA is mounted at `/` with a client-side fallback, so any route
+>    *not* under `/api` is swallowed by the frontend. The similarity router was moved to
+>    `prefix="/api/similarity"`; §9 reflects the new paths.
+> 2. **Wave 3 now runs eight agents**, not five — `citation_relevance` and `novelty`
+>    joined (`b54aebf`). The latency argument in §8.1 still holds (the wave is bounded by
+>    its slowest member and these are also network-bound), but the wave is busier than
+>    §8.1 originally described.
+> 3. **`PdfHighlightMapper` is called from `agents/report_agent.py:327`, not the
+>    orchestrator.** §8.4 said the orchestrator should make the single combined call.
+>    That is now wrong and, happily, unnecessary: `report` is wave 5 and `similarity` is
+>    wave 3, so ReportAgent already has both span sets in hand. The "call it once"
+>    requirement is satisfied structurally — Phase 5 only has to pass similarity spans
+>    into the existing call site.
+> 4. **`report_agent.py` is now 979 lines** (was 775) after the line-level AI-badge work
+>    (`7a1c4bd`, `f5f5aee`). It is well past the 800-line cap, so the Phase 7 extraction
+>    is no longer merely advisable — nothing new can go in that file until it is split.
+
+---
+
+## Table of Contents
+
+1. [What we are building](#1-what-we-are-building)
+2. [Decisions already locked](#2-decisions-already-locked)
+3. [Why this is hard (and how we get around it)](#3-why-this-is-hard-and-how-we-get-around-it)
+4. [How the number is computed](#4-how-the-number-is-computed)
+5. [Architecture — where every piece lives](#5-architecture--where-every-piece-lives)
+6. [Data model](#6-data-model)
+7. [Corpus providers](#7-corpus-providers)
+8. [Integration into the existing pipeline](#8-integration-into-the-existing-pipeline)
+9. [API surface](#9-api-surface)
+10. [Frontend](#10-frontend)
+11. [PDF report](#11-pdf-report)
+12. [Configuration](#12-configuration)
+13. [Testing strategy](#13-testing-strategy)
+14. [Evaluation and calibration](#14-evaluation-and-calibration)
+15. [Build order — phases with exit criteria](#15-build-order--phases-with-exit-criteria)
+16. [Risks and mitigations](#16-risks-and-mitigations)
+17. [What we are deliberately NOT building](#17-what-we-are-deliberately-not-building)
+18. [Open questions](#18-open-questions)
+
+---
+
+## 1. What we are building
+
+A report, attached to every analysis job, that answers: **which passages of this paper
+already exist somewhere else, and where.**
+
+Output shape, mirroring the reference screenshot:
+
+```
+Research_Proposal.pdf
+CORPUS SIMILARITY REPORT
+
+  11%              10%              8%               8%
+  CORPUS           OPEN ACCESS      PUBLICATIONS     CITATIONEDGE
+  SIMILARITY       SOURCES                           CORPUS
+
+PRIMARY SOURCES
+  [1] arxiv.org · Open Access                                    5%
+  [2] www.isca-archive.org · Open Access                         2%
+  [3] "Text, Speech and Dialogue", Springer 2022 · Publication   1%
+  [4] CitationEdge Corpus · submitted 2026-03-14                 1%
+  ...
+
+Checked against 412 candidate sources across 4 providers · 6,180 words compared
+· quoted text and bibliography excluded · not a Turnitin-equivalent index
+```
+
+Plus a **separate, non-headline** "Paraphrase Risk" signal, and **on-page highlighting**
+of matched spans over the rasterized PDF (the viewer already exists — see §8.4).
+
+### Why this feature belongs in CitationEdge
+
+Per `PRODUCT.md`, the product promise is "depth over search" and "grounded output". A
+similarity report is the natural sibling of the existing `citation_gap` agent: that one
+says *"you should have cited something here"*, this one says *"this passage already
+exists there"*. Both are claims about the paper's relationship to the literature, and
+both must trace back to a real, linkable source. The two features share the same
+provider layer, the same ranking intuition, and the same UI grammar.
+
+---
+
+## 2. Decisions already locked
+
+| # | Decision | Chosen | Why |
+|---|---|---|---|
+| 1 | Corpus | Hybrid: internal upload corpus + open academic full text | Free, no vendor lock-in, defensible provenance. Internal corpus gives duplicate-submission and self-plagiarism detection that no external API can. |
+| 2 | Match method | Verbatim fingerprinting for the headline %, semantic embeddings as a separate signal | A percentage must mean "this fraction of words is copied". Cosine similarity cannot carry that meaning — every paper scores non-zero against its own field. |
+| 3 | Framing | "Corpus Similarity" with mandatory coverage disclosure | We cannot index what Turnitin indexes. Shipping the number without the denominator would violate `PRODUCT.md` principle 2 (grounded output) and invites misconduct decisions on incomplete evidence. |
+| 4 | Integration | Pipeline agent (wave 3, non-critical) + on-demand recheck endpoint | Fits the existing DAG exactly; recheck lets a report be refreshed as the corpus grows without re-parsing the PDF. |
+
+---
+
+## 3. Why this is hard (and how we get around it)
+
+### The core problem
+
+Turnitin works because it owns a crawled index of ~100 billion web pages plus licensed
+publisher content plus ~2 billion student papers. We have none of that, and building it
+is not a feature — it is a company.
+
+**We cannot download the corpus. So we invert the problem: we let someone else's
+inverted index do the retrieval, and we do the verification locally.**
+
+### Two-stage retrieval
+
+```
+Stage 1 — RETRIEVAL (remote, cheap, recall-oriented)
+  Pick ~20-30 distinctive exact phrases from the paper.
+  Send each as a phrase query to CORE's Solr index:
+      GET https://api.core.ac.uk/v3/search/works?q=fullText:"<phrase>"
+  CORE searches 40M+ open-access full texts and returns the works
+  containing that literal phrase, WITH their full text in the response.
+  → Yields a candidate set of maybe 50-400 documents.
+
+Stage 2 — VERIFICATION (local, exact, precision-oriented)
+  For each candidate: winnowing fingerprint intersection → seed matches
+  → greedy span extension → exact character offsets in both documents.
+  → Yields defensible spans with real word counts.
+```
+
+This is the whole trick. Stage 1 costs a few HTTP calls and gives us recall against a
+corpus we could never host. Stage 2 costs milliseconds of CPU and gives us the precision
+that makes a percentage honest. **A phrase that appears in no candidate contributes 0% —
+we never estimate, interpolate, or ask an LLM to guess.**
+
+### Why phrase queries and not keyword queries
+
+A keyword query (`fullText:"transformer attention"`) returns tens of thousands of
+topically-related papers, none of which necessarily share a single sentence. A phrase
+query for a distinctive 8-12 word run returns only documents that literally contain those
+words. Recall is lower, but every hit is already close to a confirmed match, so the
+verification stage does very little wasted work. This is the same reason
+`paper_recommender.py` strips `FILLER_TOKENS` before querying — generic words return
+generic results.
+
+### Choosing which phrases to query
+
+We cannot query every phrase in the paper (a 6,000-word paper has ~6,000 overlapping
+8-grams). We select the most *distinctive* ones:
+
+1. Drop phrases that are wholly inside excluded regions (bibliography, quotes, headers).
+2. Drop phrases whose tokens are all in `STOPWORDS ∪ FILLER_TOKENS`
+   (reuse the lists already curated in `services/paper_recommender.py` — do not fork them).
+3. Drop phrases matching the **boilerplate stoplist** (§16, risk 3):
+   "in this paper we propose", "the rest of this paper is organized as follows", etc.
+4. Score remaining phrases by mean inverse token frequency *within the document* —
+   rare-in-this-paper words are usually rare globally too.
+5. Spread the selection across sections so we don't query 30 phrases from the
+   introduction and none from the methods.
+6. Take the top `SIM_MAX_QUERIES` (default 24).
+
+Reasoning: a paper that copies will almost always copy a *contiguous run*. A stratified
+sample of 24 distinctive phrases across the document has high probability of landing
+inside any copied run longer than ~150 words. Copied runs shorter than that are caught
+only if a sampled phrase happens to land in them — an acknowledged recall limit, and one
+more reason the coverage disclosure is mandatory.
+
+---
+
+## 4. How the number is computed
+
+This section is the spec. Everything else is plumbing.
+
+### 4.1 The denominator
+
+```
+total_words = words in the document AFTER exclusions
+```
+
+Excluded by default (each independently toggleable, mirroring Turnitin's options):
+
+| Region | Excluded? | Why |
+|---|---|---|
+| Bibliography / references section | Yes | Reference strings are *supposed* to be identical to the original. Including them inflates every paper by 3-8%. `ParserAgent` already isolates this section. |
+| Quoted text (`"..."`, block quotes) | Yes | Properly attributed quotation is not plagiarism. |
+| Title / author / affiliation block | Yes | Institutional addresses match thousands of papers. |
+| Equations, tables, figure captions | Yes | Notation collides across the whole field. |
+| Everything else | No | This is the paper. |
+
+### 4.2 The coverage bitmap
+
+The single data structure the whole score rests on:
+
+```python
+covered: list[int | None]   # one slot per word in the post-exclusion document
+                            # value = index of the source that owns this word, or None
+```
+
+Every confirmed match writes its word range into the bitmap. **A word is counted once no
+matter how many sources contain it** — this is what stops the percentages from exceeding
+100% when a boilerplate sentence appears in forty papers.
+
+```
+overall_similarity_pct = round(100 * count(covered[i] is not None) / total_words)
+```
+
+### 4.3 Attribution — who gets credit for a word
+
+When two sources both contain a span, the word is attributed to **the source with the
+longest confirmed match overlapping that word**; ties break on higher provider trust
+(internal corpus > CORE full text > publisher abstract). This is why per-source
+percentages in the primary-sources list *approximately* sum to the headline index rather
+than wildly exceeding it.
+
+```
+per_source_pct[s] = round(100 * count(covered[i] == s) / total_words)
+```
+
+### 4.4 Bucket percentages
+
+The three bucket numbers (Open Access / Publications / CitationEdge Corpus) are **each
+computed independently**, as "the similarity index you would get if this were the only
+bucket". They deliberately do **not** sum to the headline number — in the reference
+screenshot, 10 + 8 + 8 = 26 against an index of 11, for exactly this reason. Turnitin
+behaves the same way; replicating it avoids surprising anyone who has read a real report.
+
+```python
+bucket_pct[b] = round(100 * count(word covered by ANY source in bucket b) / total_words)
+```
+
+Implementation: run the bitmap fill three more times, once per bucket, over the same
+confirmed match list. Cheap (it is a pass over a list of integers), and much clearer than
+trying to derive bucket numbers from the attributed bitmap.
+
+### 4.5 Minimum match length
+
+A match must be at least `SIM_MIN_MATCH_WORDS` (default **8**) consecutive words to enter
+the bitmap. Below that, shared runs are linguistic coincidence, not reuse. This threshold
+is the single biggest lever on the false-positive rate and must be exposed in config and
+recorded in the report output so a result is reproducible.
+
+### 4.6 The paraphrase signal (separate, never in the %)
+
+Parallel pass using the existing `VectorStoreService` (SciBERT):
+
+- Embed each body paragraph of the uploaded paper.
+- Embed each candidate document's paragraphs (candidates are already in hand from stage 1
+  — no extra network cost).
+- Flag pairs with cosine ≥ `SIM_PARAPHRASE_THRESHOLD` (default 0.92) **that are not
+  already verbatim-matched**.
+
+Reported as a count and a list — "4 passages closely track a source without verbatim
+overlap" — with the passage pairs shown side by side. It is a *reading prompt for a
+human*, not a score, and it never touches the headline number. Reasoning: SciBERT cosine
+between two paragraphs from the same subfield routinely hits 0.85 with zero copying; any
+percentage built on that would be noise wearing a lab coat.
+
+---
+
+## 5. Architecture — where every piece lives
+
+The new code goes in a **self-contained `similarity/` package**, deliberately mirroring
+the existing `text_detection/` package (config / schema / pipeline / pluggable backends,
+everything env-toggled, degrades instead of blocking). That package is the house style
+for "a bounded analytical capability the pipeline calls into", and copying its shape
+means anyone who has read one can read the other.
+
+```
+similarity/
+├── __init__.py
+├── config.py               # pydantic-settings, all SIM_* env vars          (~80 lines)
+├── schema.py               # Pydantic models — the contract                 (~150)
+├── normalize.py            # canonicalization + offset map                  (~120)
+├── exclusions.py           # bibliography / quotes / boilerplate regions    (~180)
+├── fingerprint.py          # k-gram hashing + winnowing                     (~130)
+├── selector.py             # pick distinctive query phrases                 (~140)
+├── matcher.py              # fingerprint seeds → verified spans             (~200)
+├── aggregate.py            # coverage bitmap → percentages + buckets        (~180)
+├── semantic.py             # SciBERT paraphrase pass                        (~120)
+├── pipeline.py             # orchestrates all of the above                  (~220)
+├── index_writer.py         # write this doc into the internal corpus        (~110)
+└── corpus/
+    ├── __init__.py
+    ├── base.py             # CorpusProvider protocol + Candidate dataclass  (~90)
+    ├── core_api.py         # CORE v3 — the full-text workhorse              (~170)
+    ├── openalex.py         # abstract-level matching                        (~90)
+    ├── arxiv.py            # abstract + OA PDF fetch                        (~110)
+    ├── crossref.py         # publisher metadata / abstracts                 (~90)
+    └── internal.py         # prior CitationEdge uploads                     (~190)
+```
+
+Every file lands well inside the 800-line cap and most inside the 200-400 "typical" band
+from the coding-style rules.
+
+### Files modified (not created)
+
+| File | Current | Change | Note |
+|---|---|---|---|
+| `agents/similarity_agent.py` | — | **new**, ~90 lines | Thin adapter: pull text from Neo4j → call pipeline → write Neo4j + return dict. All logic lives in `similarity/`. |
+| `agents/__init__.py` | | export `SimilarityAgent` | |
+| `orchestrators/custom_orchestrator.py` | 359 | +~40 | Register agent; fetch similarity rows; add to `final_result` / `save_result`. |
+| `configs/pipeline_config.yaml` | 39 | +4 | Declare `similarity` at wave 3, `depends_on: [parser, keyword]`. |
+| `services/pdf_highlight_mapper.py` | 307 | ~+30 | Generalize hardcoded `"ai"` / `"human"` typing — see §8.4. |
+| `backend/routers/reports.py` | 478 | +~45 | `GET /reports/{id}/similarity`. |
+| `backend/routers/similarity.py` | — | **new**, ~110 | `POST /{id}/recheck`, `GET /status`. |
+| `backend/schemas/similarity.py` | — | **new**, ~40 | Request + status models; reuse `similarity.schema` for the response, exactly as `text_detection` does. |
+| `backend/main.py` | 288 | +2 | Register router (done). |
+| `agents/report_agent.py` | **979** | **extract, then +2** | Grew from 775 to 979 in the merge; far past the cap and cannot absorb a new section. Extract the similarity section into `agents/report_sections/similarity_section.py`. |
+| `frontend-app/src/pages/ReportPage.jsx` | **869** | **−~300, +~8** | **Already over the 800 cap.** Extract `buildReport()` and `DEMO` into `pages/reportAdapter.js` / `pages/reportDemo.js` first, then add the similarity block as 8 lines calling new components. |
+| `frontend-app/src/components/SimilarityIndex.jsx` + `.css` | — | **new**, ~160 | Header block: big %, three buckets. |
+| `frontend-app/src/components/SimilaritySources.jsx` | — | **new**, ~140 | Numbered, colour-coded primary sources. |
+| `frontend-app/src/components/CoverageNote.jsx` | — | **new**, ~60 | The disclosure. |
+| `frontend-app/src/api.js` | 103 | +~15 | `getSimilarity`, `recheckSimilarity`. |
+| `requirements.txt` | 56 | +0 | **No new dependencies.** `httpx`, `numpy`, `pymupdf`, `transformers` are all present. |
+
+**Zero new runtime dependencies** is a deliberate design constraint: the fingerprinting is
+~130 lines of standard-library Python, and adding a plagiarism SDK would undo the
+"no vendor lock-in" decision from §2.
+
+---
+
+## 6. Data model
+
+### 6.1 Pydantic schema (`similarity/schema.py`)
+
+```python
+class Bucket(str, Enum):
+    OPEN_ACCESS   = "open_access"     # CORE, arXiv — the "Internet Sources" analogue
+    PUBLICATION   = "publication"     # Crossref/DOI-bearing, publisher-hosted
+    INTERNAL      = "internal"        # prior CitationEdge uploads
+
+class MatchSpan(BaseModel):
+    doc_start_word: int               # inclusive, index into post-exclusion word list
+    doc_end_word: int                 # exclusive
+    doc_char_start: int               # offset into ORIGINAL text — drives highlighting
+    doc_char_end: int
+    word_count: int
+    excerpt: str                      # <= 300 chars, for display
+    source_excerpt: str               # the matching text in the source
+    source_index: int                 # 1-based, matches the numbered UI chips
+
+class SourceMatch(BaseModel):
+    source_index: int
+    bucket: Bucket
+    title: str
+    url: str                          # ALWAYS resolvable — a source you cannot open
+                                      # is not evidence. Candidates without a URL are
+                                      # dropped, same rule as _score_candidates().
+    doi: str | None
+    authors: list[str] = []
+    year: int | None
+    provider: str                     # "CORE" | "arXiv" | "Crossref" | "OpenAlex" | "CitationEdge Corpus"
+    display_label: str                # "arxiv.org" / '"Text, Speech and Dialogue", Springer, 2022'
+    matched_words: int                # attributed words only (§4.3)
+    percent: int                      # matched_words / total_words * 100, rounded
+    spans: list[MatchSpan]
+
+class ParaphraseFlag(BaseModel):
+    doc_excerpt: str
+    source_excerpt: str
+    source_index: int
+    cosine: float
+
+class Coverage(BaseModel):
+    """Mandatory disclosure — the denominator behind the number."""
+    total_words_compared: int
+    words_excluded: int
+    exclusions_applied: list[str]       # ["bibliography", "quotes", "title_block", ...]
+    providers_queried: list[str]
+    providers_failed: list[str]         # a 429 from CORE must be visible, never silent
+    phrases_queried: int
+    candidates_retrieved: int
+    candidates_verified: int
+    internal_corpus_size: int
+    min_match_words: int
+    budget_exhausted: bool              # True if we stopped early on the time budget
+    checked_at: datetime
+
+class SimilarityReport(BaseModel):
+    overall_percent: int
+    bucket_percents: dict[Bucket, int]
+    sources: list[SourceMatch]          # sorted by percent desc; the "primary sources" list
+    paraphrase_flags: list[ParaphraseFlag]
+    coverage: Coverage
+    status: Literal["complete", "partial", "unavailable"]
+    notes: list[str]                    # human-readable caveats surfaced in the UI
+    processing_time_ms: float
+```
+
+`status` semantics, following the `text_detection` precedent of degrading rather than
+blocking:
+
+- `complete` — all configured providers answered.
+- `partial` — at least one provider failed or the time budget was exhausted. The number
+  is a **lower bound**; the UI must say so.
+- `unavailable` — no provider answered, or the document was too short
+  (`< SIM_MIN_DOCUMENT_WORDS`, default 300). **No number is shown.** We never render a
+  0% that means "we didn't look".
+
+### 6.2 Persistence
+
+| Store | Collection / label | Contents | Why there |
+|---|---|---|---|
+| MongoDB | `results.similarity` | Full `SimilarityReport` dict | Same place every other agent output lives; `GET /reports/{id}/json` picks it up for free via the existing `$set` merge in `save_result`. |
+| MongoDB | `corpus_fingerprints` | `{doc_id, job_id, title, owner_email, word_count, fingerprints: [int], created_at}` | The internal corpus inverted index. Multikey index on `fingerprints` makes candidate lookup a single indexed query. |
+| MongoDB | `corpus_texts` | `{doc_id, normalized_text, offset_map}` | Needed to verify and excerpt an internal match. Separate collection so the fingerprint index stays small and hot. |
+| Neo4j | `(:Document)-[:HAS_SIMILARITY]->(:SimilarityReport)` and `(:SimilarityReport)-[:MATCHES]->(:SimilaritySource)` | Summary properties only | Keeps the graph the primary thinking surface (`PRODUCT.md` principle 4) — you can traverse from a paper to the works it overlaps. Full spans stay in Mongo; the graph is not a document store. |
+| LanceDB | table `corpus_paragraphs` | Paragraph embeddings for the paraphrase pass | `LanceDBService` already exists and is already wired into `AgentContext.vector_db`. |
+
+---
+
+## 7. Corpus providers
+
+All providers implement one protocol so the pipeline is provider-agnostic and each can be
+switched off independently:
+
+```python
+class CorpusProvider(Protocol):
+    name: str
+    bucket: Bucket
+    async def search_phrase(self, phrase: str, limit: int) -> list[Candidate]: ...
+    async def fetch_text(self, candidate: Candidate) -> str | None: ...
+```
+
+### 7.1 CORE — the workhorse
+
+- Endpoint: `GET https://api.core.ac.uk/v3/search/works?q=fullText:"<phrase>"`
+- Coverage: ~300M metadata records, **40M+ full texts** — the largest open-access
+  aggregation available.
+- Auth: free; works unauthenticated but **register for an API key** — the unregistered
+  limit is roughly 5 single requests / 10 seconds, which 24 phrase queries would blow
+  through in the first two seconds.
+- Crucially, the response can carry `fullText`, so retrieval and text-fetch collapse into
+  one round trip.
+- Handling: shared `asyncio.Semaphore(2)` plus a stagger, exactly like `_PROVIDER_SEM` in
+  `paper_recommender.py`; `@async_retry` on 429/5xx; on persistent failure record it in
+  `coverage.providers_failed` and continue.
+
+### 7.2 arXiv
+
+- Reuse `_search_arxiv()` from `services/paper_recommender.py` verbatim for metadata.
+- For candidates that survive abstract-level screening, fetch the OA PDF and extract text
+  with the existing `utils.pdf.extract_text_from_pdf`. Cap at `SIM_MAX_PDF_FETCHES`
+  (default 5) — PDF fetch and parse is the slowest thing in the pipeline.
+- Bucket: `OPEN_ACCESS`.
+
+### 7.3 OpenAlex and Crossref
+
+- Reuse `_search_openalex()` and `_search_crossref()` unchanged.
+- **Abstracts only** — neither serves full text. An abstract-level match is real but
+  small, so these mostly populate the `PUBLICATION` bucket with 1% entries, which is
+  exactly what the reference screenshot shows for Springer.
+- `source_for()` and the `DOI_PUBLISHER` prefix map already turn a DOI into
+  "Elsevier" / "Springer" / "IEEE" — reuse directly for `display_label`.
+
+**DRY note:** these three provider functions are imported from `paper_recommender.py`,
+never copied. If that means lifting them into a shared `services/academic_search.py` that
+both modules import, do that refactor as part of Phase 2 — it is a targeted improvement
+to code we are already working in, not unrelated cleanup.
+
+### 7.4 Internal corpus
+
+The only bucket where we own the index, and the only one that can catch a paper
+submitted twice.
+
+**Write path** (`similarity/index_writer.py`, called at the end of a successful job when
+`SIM_INDEX_UPLOADS=true`): normalize → fingerprint → insert into `corpus_fingerprints` +
+`corpus_texts` → upsert paragraph embeddings into LanceDB.
+
+**Read path:** fingerprint the new document → query
+`corpus_fingerprints.find({fingerprints: {$in: doc_fingerprints}})` → the multikey index
+returns only documents sharing at least one fingerprint → fetch their normalized text →
+verify locally. No embeddings needed for the verbatim path; this is exact and fast.
+
+**Privacy — this is not optional.** Another researcher's uploaded paper is confidential.
+The report may show:
+
+- that a match exists,
+- the matched **excerpt from the user's own document**,
+- a neutral label: `CitationEdge Corpus · submitted 2026-03-14`,
+- the percentage.
+
+The report must **never** expose the other document's title, authors, owner, full text, or
+a link to it. This mirrors how Turnitin handles student-paper matches, and it is the
+behaviour that lets us keep the bucket at all. Enforcement: `SourceMatch.title` and
+`.url` are set to the neutral label for `Bucket.INTERNAL`, in `internal.py`, at
+construction — not filtered later in the UI where a future refactor could drop the filter.
+
+Self-match guard: a document must never match itself. Exclude by `doc_id`, and on
+recheck, by `job_id` too.
+
+---
+
+## 8. Integration into the existing pipeline
+
+### 8.1 The agent
+
+`agents/similarity_agent.py`:
+
+```python
+class SimilarityAgent(BaseAgent):
+    name = "similarity"
+    wave = 3          # after parser (w1) and keyword (w2)
+    critical = False  # a similarity failure must never fail an analysis
+```
+
+**Why wave 3.** It needs paragraphs from `ParserAgent` (wave 1) and benefits from
+`KeywordAgent` output (wave 2) for query construction and domain-token scoring. Wave 3
+already runs five agents in parallel — `citation_gap`, `claim_verifier`,
+`counter_factuality`, `evidence_grounding`, `argumentation` — and all but the last are
+already blocking on external HTTP. `HybridWorkflow` runs a wave via `ParallelWorkflow`,
+so the added wall-clock cost is `max(0, similarity_time − current_wave3_time)`, which for
+a 45-second budget against agents that already take that long is close to zero.
+
+**Why `critical = False`.** Look at `HybridWorkflow.execute`: a failed critical agent
+breaks the wave loop and aborts the pipeline. A CORE outage must not cost the user their
+claim verification, citation gaps, and PDF report. Non-critical failure degrades the job
+to `partial_failure` and the rest of the analysis completes — the same choice
+`citation_gap`, `claim_verifier`, `counter_factuality`, and `evidence_grounding` already
+make.
+
+The agent itself stays thin (~90 lines): read paragraphs from Neo4j (the same query the
+orchestrator already uses for AI-text detection), call
+`SimilarityPipeline.run(text, doc_id, keywords)`, write summary nodes to Neo4j, return
+the report dict. All algorithmic work lives in `similarity/` where it can be unit-tested
+without a database.
+
+### 8.2 Time budget
+
+A hard `asyncio.wait_for` wrapping the whole retrieval stage at `SIM_BUDGET_SECONDS`
+(default 45). On timeout the pipeline returns whatever it has verified so far with
+`status="partial"` and `coverage.budget_exhausted=True`. Partial evidence, honestly
+labelled, beats a hung job — and this is what makes it safe to put a network-bound step
+inside a user-facing pipeline at all.
+
+### 8.3 Orchestrator wiring
+
+In `custom_orchestrator.py`, following the exact pattern already used for
+`counterfactuality` and `ai_text_detection`:
+
+1. Add `SimilarityAgent()` to `get_agents()`.
+2. After the workflow returns, read the similarity result off the `AgentResult` list
+   (cleaner than a Neo4j round trip, and the report is a document not a graph).
+3. Add `similarity` to `final_result`, to `upsert_job`, and to `save_result`.
+
+Note that `save_result` merges with `$set`, so adding a key is non-breaking for existing
+consumers.
+
+### 8.4 Highlighting — reuse, with one required change
+
+`services/pdf_highlight_mapper.py` already does the hard part: PyMuPDF word extraction,
+normalized substring location, multi-line box grouping, 0-1 coordinate normalization, 2x
+page rasterization, and graceful handling of unmatched spans and missing text layers.
+`ReportPage.jsx` already renders those boxes over page images served by
+`GET /jobs/{id}/pages/{n}.png`. **We should not build a second highlighting system.**
+
+Two changes are required:
+
+1. **Generalize the span type.** `_match_span()` hardcodes
+   `"type": "ai" if span.get("verdict") == "AI_GENERATED" else "human"`, and `run()`
+   filters to `verdict in ("AI_GENERATED", "REAL")`. Add an optional `kind` field on the
+   input span that passes through when present, plus an optional `meta` dict (carrying
+   `source_index` so the frontend can colour-code by source the way the reference
+   screenshot numbers and colours each entry). Default behaviour unchanged — existing
+   AI-detection callers and their tests keep working.
+
+2. **Call the mapper once, not twice.** It rasterizes into `pages_dir` and returns a full
+   `pages` array. Two independent callers would either race on the PNGs or produce two
+   `page_highlights` structures that the frontend cannot merge.
+
+   **Corrected after the merge:** the sole caller is `agents/report_agent.py:327`, not the
+   orchestrator as this section originally assumed. That is the easier situation —
+   `report` is wave 5 and `similarity` is wave 3, so ReportAgent already has both the AI
+   spans and the similarity report available when it builds the call. Phase 5 therefore
+   does not introduce a second call site; it extends the existing one with the similarity
+   spans, each tagged with its `kind`. The frontend then filters by `kind` to toggle
+   "AI detection" and "Similarity" overlay layers independently.
+
+This is a real integration cost and it is the reason to do highlighting in its own phase
+(Phase 5) rather than smuggling it into the agent phase.
+
+---
+
+## 9. API surface
+
+Following the conventions in `backend/backend.md`. **All routes sit under `/api`** — the
+SPA is mounted at `/` with a catch-all fallback, so a route outside `/api` returns the
+frontend's HTML instead of JSON. Each router declares its own prefix; there is no
+app-level prefix to inherit.
+
+| Method | Path | Response | Notes |
+|---|---|---|---|
+| `GET` | `/api/reports/{job_id}/similarity` | `200 SimilarityReport` · `202` not ready · `404` no job | Lives in `reports.py` beside `/json`, `/pdf`, `/graph`. |
+| `POST` | `/api/similarity/{job_id}/recheck` | `202 {job_id, status: "rechecking"}` | Re-runs against the current corpus using the stored normalized text — **no PDF re-parse, no re-analysis**. Runs as a `BackgroundTask`, same as `POST /analyze`. |
+| `GET` | `/api/similarity/status` | `200 SimilarityStatus` | Which providers are configured, corpus size, thresholds. Directly modelled on `GET /text-detection/status`, and for the same reason: it is how an operator distinguishes "nothing configured" from "configured but failing". |
+
+`recheck` needs a guard: reject with `409` if a recheck for that job is already in flight,
+tracked with a simple in-process set keyed by `job_id`. Without it, a user clicking the
+button four times launches four concurrent 24-query provider storms and earns a rate-limit
+ban for everyone.
+
+Auth: the existing routers are unauthenticated and jobs are filtered by `user_email` as a
+query parameter. Match the existing pattern — do not invent a new auth story inside this
+feature. (Flagged in §18 as a pre-existing gap worth its own work.)
+
+---
+
+## 10. Frontend
+
+### 10.1 Prerequisite refactor
+
+`ReportPage.jsx` is **869 lines — already over the 800-line cap** in the coding-style
+rules (826 before the merge; the AI-badge work added to it). Before adding anything:
+
+- Move `buildReport()` and its helpers → `pages/reportAdapter.js` (~180 lines).
+- Move the `DEMO` constant → `pages/reportDemo.js` (~90 lines).
+
+That brings `ReportPage.jsx` to roughly 590 lines and makes room. This is exactly the
+"targeted improvement to code you are working in" case — it is required to add the
+feature cleanly, not opportunistic refactoring.
+
+### 10.2 New components
+
+**`SimilarityIndex.jsx`** — the header block from the screenshot: one large percentage
+plus three bucket figures. Follows `DESIGN.md` / `PRODUCT.md` brand rules: obsidian
+background, glass card, Instrument Serif for the big number, DM Mono for the bucket
+figures. **Restraint over alarm** — the number is set in the standard white display
+treatment, not red. Red is reserved for errors in this brand, and colouring a similarity
+score red makes an accusation the data does not support. A single accent colour appears
+only on the per-source chips, to tie a source to its highlight colour on the page.
+
+**`SimilaritySources.jsx`** — the numbered list. Each row: colour chip with index, title
+as a link (internal-corpus rows are deliberately not links), provider label, percentage.
+Clicking a row scrolls the page viewer to that source's first highlight and dims the
+others.
+
+**`CoverageNote.jsx`** — always rendered, never behind a disclosure triangle:
+
+> Checked 6,180 words against 412 candidate sources from CORE, arXiv, OpenAlex, Crossref
+> and 1,204 papers in the CitationEdge corpus. Quoted text and bibliography excluded.
+> Matches under 8 consecutive words are not counted. This is not a Turnitin-equivalent
+> index — coverage is limited to open-access literature and papers analysed here.
+
+When `status === "partial"`, prepend: *"One or more sources could not be reached; this
+figure is a lower bound."* When `unavailable`, the component replaces the whole block —
+**no percentage is rendered at all.**
+
+### 10.3 Wiring
+
+`ReportPage.jsx` gains ~8 lines: read `result.similarity`, render the three components,
+pass `kind === 'similarity'` highlights to the existing page viewer. `api.js` gains
+`getSimilarity(jobId)` and `recheckSimilarity(jobId)` following the existing helper style.
+
+---
+
+## 11. PDF report
+
+`agents/report_agent.py` is **979 lines** after the merge — already past the cap. Nothing
+new goes in inline; the extraction below is now a precondition, not a preference.
+
+- Create `agents/report_sections/similarity_section.py` (~140 lines) exporting
+  `build_similarity_section(report: dict) -> list[Flowable]` returning ReportLab
+  flowables.
+- `report_agent.py` imports it and appends the result — a two-line change. Consider
+  extracting the existing AI-badge section the same way while you are in there; at 979
+  lines the file is the worst offender in the repo.
+- Section contents: the index block, the primary-sources table, and the coverage note
+  **verbatim from the UI**. The disclosure must survive the transition to PDF, because
+  the PDF is the artefact that gets emailed, forwarded, and attached to decisions.
+- Ordering: place it after AI-text detection and before citation gaps — the two
+  provenance signals belong together.
+
+`report` is wave 5 and `similarity` is wave 3, so the data is guaranteed present. When
+similarity is `unavailable`, the section renders a one-line "not available for this
+document" note rather than being omitted, so a reader can tell the check ran.
+
+---
+
+## 12. Configuration
+
+`similarity/config.py`, pydantic-settings, mirroring `text_detection/config.py`.
+
+| Env var | Default | Purpose |
+|---|---|---|
+| `SIM_ENABLED` | `true` | Master switch. |
+| `SIM_CORE_API_KEY` | `""` | CORE key. Empty ⇒ unauthenticated (heavily rate-limited) — logged as a warning at startup. |
+| `SIM_ENABLE_CORE` | `true` | Per-provider toggles… |
+| `SIM_ENABLE_ARXIV` | `true` | |
+| `SIM_ENABLE_OPENALEX` | `true` | |
+| `SIM_ENABLE_CROSSREF` | `true` | |
+| `SIM_ENABLE_INTERNAL` | `true` | |
+| `SIM_INDEX_UPLOADS` | `false` | **Ships off.** Indexing users' papers into a shared corpus is a consent decision, not a default. See §16 risk 6. |
+| `SIM_KGRAM_SIZE` | `5` | Winnowing k, in words. |
+| `SIM_WINDOW_SIZE` | `4` | Winnowing w. Guarantees detection of any shared run ≥ `w + k − 1` = 8 words. |
+| `SIM_MIN_MATCH_WORDS` | `8` | Minimum reportable match. |
+| `SIM_MIN_DOCUMENT_WORDS` | `300` | Below this ⇒ `unavailable`. |
+| `SIM_MAX_QUERIES` | `24` | Phrase queries per document. |
+| `SIM_MAX_CANDIDATES` | `400` | Candidates verified per document. |
+| `SIM_MAX_PDF_FETCHES` | `5` | OA PDFs downloaded per document. |
+| `SIM_BUDGET_SECONDS` | `45` | Hard wall-clock cap on retrieval. |
+| `SIM_EXCLUDE_QUOTES` | `true` | |
+| `SIM_EXCLUDE_BIBLIOGRAPHY` | `true` | |
+| `SIM_PARAPHRASE_THRESHOLD` | `0.92` | SciBERT cosine for the paraphrase flag. |
+| `SIM_ENABLE_PARAPHRASE` | `true` | |
+
+Every threshold that affects a number is echoed into `Coverage` so any report is
+reproducible from its own output.
+
+---
+
+## 13. Testing strategy
+
+TDD per the project rules: test first (RED), minimal implementation (GREEN), refactor,
+80% minimum coverage. The architecture is built around making this possible — the entire
+scoring core is pure functions over strings and lists, with zero I/O.
+
+### Unit — `tests/similarity/`
+
+| File | What it pins down |
+|---|---|
+| `test_normalize.py` | Offset map round-trips: for every word in the normalized text, the recorded char offsets slice the *original* text back to the same word. This is what makes highlighting land on the right pixels. |
+| `test_fingerprint.py` | **Property test:** for random text pairs sharing a run of ≥ `w+k−1` words, the fingerprint sets always intersect. This is the winnowing correctness guarantee and it is the foundation of every recall claim in this document. |
+| `test_exclusions.py` | Bibliography, quotes, and title block are removed from the denominator; a paper that is 100% bibliography yields `unavailable`, not 100%. |
+| `test_selector.py` | Phrases are stratified across sections; boilerplate and all-stopword phrases are never selected. |
+| `test_matcher.py` | Seed → span extension produces exact word and char boundaries; overlapping seeds merge into one span, not two. |
+| `test_aggregate.py` | The scoring spec in §4, case by case: identical text ⇒ 100%; disjoint text ⇒ 0%; **one span claimed by three sources counts once** (the double-count regression); attribution goes to the longest match; bucket percentages are independent and may exceed the headline; matches below `min_match_words` are dropped. |
+| `test_semantic.py` | Paraphrase flags never alter `overall_percent` — asserted directly, because this is the invariant most likely to be broken by a future "improvement". |
+| `test_privacy.py` | An internal-corpus `SourceMatch` never carries the other document's title, URL, authors, or text. Asserted at the boundary so a UI change cannot leak it. |
+
+### Provider tests — no network
+
+`httpx.MockTransport` with recorded fixtures for CORE / arXiv / OpenAlex / Crossref,
+covering: happy path, 429, 5xx, malformed JSON, empty results, and a candidate with no
+resolvable URL. Matches the existing "no real DB or LLM required" testing posture in
+`backend/tests/conftest.py`.
+
+### Integration — `backend/tests/test_similarity.py`
+
+Router behaviour with a mocked pipeline, via `app.dependency_overrides`: report retrieval,
+202 while pending, 404 unknown job, recheck accepted, recheck 409 when already in flight,
+status endpoint shape.
+
+### Pipeline test
+
+One end-to-end test with a stub provider returning a known document: assert the pipeline
+produces exactly the expected percentage. This is the test that catches integration drift
+between the seven modules.
+
+---
+
+## 14. Evaluation and calibration
+
+A similarity number nobody has measured is a guess with a decimal point.
+`scripts/eval_similarity.py`, modelled on the existing `scripts/eval_text_detection.py`:
+
+**Construction.** Take N open-access papers with known full text from CORE. Build
+synthetic documents with *known* copy rates: 0%, 5%, 10%, 25%, 50% verbatim insertion from
+other OA papers, at varying run lengths (10 / 50 / 200 words). Also build a paraphrased
+set (LLM-reworded insertions) to measure how much verbatim matching misses by design.
+
+**Metrics.**
+
+| Metric | Target | Why |
+|---|---|---|
+| Mean absolute error vs. true copy rate | ≤ 3 points at 200-word runs | The headline claim. |
+| False-positive rate on the 0% set | **< 1%** | The number that matters most. A clean paper reading 6% destroys trust permanently. |
+| Recall by run length | reported, not gated | Honest characterisation of the sampling limit from §3. |
+| Paraphrase recall | reported | Quantifies what the verbatim index cannot see, and sizes the paraphrase signal. |
+| p50 / p95 latency | p95 ≤ `SIM_BUDGET_SECONDS` | Confirms the budget is real. |
+
+**Calibration.** Tune `SIM_MIN_MATCH_WORDS` and the boilerplate stoplist against the 0%
+set until FPR < 1%, *then* measure MAE. Precision first: an inflated score on an honest
+paper is a much worse failure than a missed match on a dishonest one. Commit the results
+to `similarity_eval_report.md` alongside the existing `agent_accuracy_report.md`.
+
+---
+
+## 15. Build order — phases with exit criteria
+
+Each phase is independently shippable and independently testable. Phases 0-1 deliver a
+working, provable feature with no external dependency at all — which is what makes the
+external-provider work in Phase 2 low-risk.
+
+| Phase | Work | Exit criteria | Rough size |
+|---|---|---|---|
+| ✅ **0. Scoring core** | `schema`, `normalize`, `exclusions`, `fingerprint`, `selector`, `matcher`, `aggregate`. Pure functions, no I/O. | All unit tests green incl. the winnowing property test. Feeding a document against itself yields exactly 100%; against unrelated text, 0%. | ~1,000 lines + tests |
+| ✅ **1. Internal corpus** | `corpus/internal.py`, `index_writer.py`, Mongo collections + indexes, privacy boundary, corpus purge on job delete. | Upload paper A, then upload A again ⇒ ~100% against the internal corpus, with the other document's identity **not** in the response. Privacy test green. | ~300 |
+| ✅ **2. External providers** | `corpus/{base,core_api,arxiv,openalex,crossref}.py`. Lift shared search into `services/academic_search.py`. | Mocked-transport tests green. Manual run against a paper with a known arXiv preprint finds it. | ~550 |
+| ✅ **3. Pipeline + agent** | `pipeline.py`, `semantic.py`, `SimilarityAgent`, orchestrator wiring, budget, persistence. | Full analysis job produces `results.similarity`; killing CORE mid-run yields `status="partial"` and a completed job. | ~450 |
+| ✅ **4. API** | `reports.py` extension, `routers/similarity.py`, schemas, recheck guard. | Integration tests green; `/docs` shows the endpoints. | ~200 |
+| ⬜ **5. Highlighting** | Generalize `PdfHighlightMapper`; single combined mapper invocation. | Similarity spans render as boxes on real page images; existing AI-detection highlights unchanged and their tests still pass. | ~120 |
+| ⬜ **6. Frontend** | `ReportPage.jsx` extraction refactor, then the three new components + `api.js`. | Report page renders the block; `ReportPage.jsx` back under 800 lines; `unavailable` renders no percentage. | ~450 |
+| ⬜ **7. PDF section** | `report_sections/similarity_section.py`. | Generated PDF contains the block and the coverage note. | ~150 |
+| ⬜ **8. Eval + docs** | `scripts/eval_similarity.py`, calibration, `backend.md` + `README.md` updates. | FPR < 1% on the 0% set; `similarity_eval_report.md` committed. | ~300 |
+
+**Phases 0–4 are done and merged** (commits `12cfcb6`, `69c39cd`, `658a727`, `ecc8207`,
+`328d8c2`, merged at `506ad88`). Post-merge suite: **175 passed, 2 failed, 1 skipped** —
+both failures are `frontend-app/dist` missing in the test environment
+(`test_root`, `test_spa_served_on_client_side_routes`), unrelated to similarity.
+
+**Suggested checkpoints:** review after Phase 1 (the scoring is provable and needs no
+network — this is where a wrong percentage formula is cheapest to fix), and again after
+Phase 3 (the first time it touches a real job).
+
+---
+
+## 16. Risks and mitigations
+
+**1. The number is read as a misconduct verdict.**
+*Impact: severe — someone's career.* This is the dominant risk and it is not technical.
+Mitigations: the "Corpus Similarity" label instead of "Similarity Index"; mandatory
+coverage disclosure in UI *and* PDF; no red/alarm styling; no threshold that produces a
+pass/fail; language throughout is "passages to review", never "plagiarism detected".
+`status="unavailable"` renders no number at all.
+
+**2. Preprint self-match.** A published paper will match its own arXiv preprint at ~100%
+and produce a terrifying, meaningless score. **This is the single most likely first bug
+report.** Mitigation: after retrieval and before scoring, drop any candidate whose
+normalized title has ≥ 0.9 token overlap with the uploaded paper's title, or whose DOI
+matches, or whose author set overlaps by ≥ 50% while similarity is > 60%. Surface dropped
+candidates in `coverage.notes` as "1 probable preprint/self-citation excluded" so the
+exclusion is visible rather than mysterious. Needs its own test fixture.
+
+**3. Boilerplate false positives.** "The rest of this paper is organized as follows",
+standard method descriptions, dataset names, common equations. Mitigation: the
+`SIM_MIN_MATCH_WORDS=8` floor; a curated boilerplate stoplist seeded from the eval corpus
+(any 8-gram appearing in > 0.5% of sampled papers); exclusion of equations and captions.
+Measured directly by the FPR metric in §14.
+
+**4. CORE rate limiting.** Unregistered access is ~5 requests / 10s. Mitigations: register
+for a key; shared semaphore + stagger (the `_PROVIDER_SEM` pattern already in the
+codebase); `@async_retry` with backoff; phrase-query result caching keyed by phrase hash;
+the hard time budget; failures recorded in `coverage.providers_failed`, never swallowed.
+
+**5. Pipeline latency.** Mitigations: wave 3 parallelism, `critical=False`, the 45s hard
+budget, and partial results. Worst case the wave takes 45s instead of ~40s.
+
+**6. Corpus consent.** Indexing a user's paper so it can be matched against other users'
+papers is a meaningful commitment about their unpublished work. Mitigations:
+`SIM_INDEX_UPLOADS` **ships `false`**; enabling it is a deployment decision that requires
+a corresponding change to user-facing terms; fingerprints and normalized text are stored,
+and the privacy boundary in §7.4 means a match never reveals whose paper it hit. A
+deletion path (`DELETE /jobs/{id}` must also purge `corpus_fingerprints` and
+`corpus_texts`) is required before this is switched on — **built in Phase 1**.
+
+**7. Scanned PDFs with no text layer.** No text ⇒ no fingerprints ⇒ 0%, which reads as
+"clean" and is actually "we couldn't look". Mitigation: `SIM_MIN_DOCUMENT_WORDS` forces
+`unavailable`. `PdfHighlightMapper` already flags `hasTextLayer=false` per page; surface
+that in `coverage.notes`. `VisualParserAgent` exists but is figure-oriented — OCR fallback
+is out of scope here.
+
+**8. Non-English papers.** Normalization is Latin-script-oriented and the corpus skews
+English. `PRODUCT.md` already lists non-English support as undecided. Mitigation: detect
+and note in `coverage.notes`; do not silently return a meaningless low score.
+
+---
+
+## 17. What we are deliberately NOT building
+
+Per YAGNI, and to keep the first version reviewable:
+
+- **A web crawler.** The "Internet Sources" bucket is open-access academic literature, not
+  the web. Anything else is a dishonest label.
+- **A commercial API fallback.** Explicitly rejected in §2. The `CorpusProvider` protocol
+  means one could be added later as one more provider — but only as a deliberate decision,
+  not a hedge.
+- **Cross-language similarity.** Requires translation of the whole corpus.
+- **Source-code similarity.** Different tokenization, different product.
+- **An instructor/reviewer dashboard, class management, submission windows.** Turnitin is
+  a workflow product; this is an analysis feature.
+- **Auto-generated misconduct reports.** We surface evidence. A human decides.
+- **An LLM in the scoring path.** The percentage must be reproducible and explainable. An
+  LLM may later *summarize* a report; it must never *compute* one.
+
+---
+
+## 18. Open questions
+
+Not blocking — each has a stated default so implementation can proceed.
+
+1. **CORE API key** — needs registering at `core.ac.uk/services/api`. *Default: build and
+   test unauthenticated with a low `SIM_MAX_QUERIES`; register before Phase 8 eval, which
+   will otherwise take hours.*
+2. **Corpus consent copy** — if `SIM_INDEX_UPLOADS` is ever switched on, the upload page
+   needs a line about it. *Default: ships off; revisit when enabling.*
+3. **Recheck rate limiting per user** — the in-process guard prevents concurrent rechecks
+   of the same job, not a user rechecking forty jobs. *Default: ship the per-job guard;
+   add per-user throttling if it becomes a problem.*
+4. **Auth on the new endpoints** — the existing routers are unauthenticated and filter by
+   `user_email` as a query param, which means anyone with a `job_id` can read a report.
+   Similarity data is more sensitive than a keyword list. *Default: match the existing
+   pattern for consistency; flag it as a pre-existing gap deserving its own piece of work
+   rather than a divergent auth story invented inside this feature.*
+
+---
+
+## Appendix — key source references
+
+- Schleimer, Wilkerson & Aiken (2003), *Winnowing: Local Algorithms for Document
+  Fingerprinting* — the algorithm behind MOSS, and behind §4's detection guarantee that
+  any shared run of ≥ `w + k − 1` tokens is found.
+- [CORE API v3](https://core.ac.uk/services/api) — ~300M metadata records, 40M+ full
+  texts, free, `fullText:"..."` phrase query support.
+  [Docs](https://api.core.ac.uk/docs/v3).
+- [OpenAlex works search](https://docs.openalex.org/api-entities/works/search-works) —
+  already integrated in `services/paper_recommender.py`.
+- Existing code this plan builds on, and must not duplicate:
+  `services/paper_recommender.py` (providers, tokenization, ranking),
+  `services/pdf_highlight_mapper.py` (PDF coordinate mapping),
+  `services/vector_store_service.py` (SciBERT),
+  `services/lancedb_service.py` (vector store),
+  `text_detection/` (package shape, degradation posture, status endpoint),
+  `workflows/hybrid_workflow.py` (wave semantics and critical-agent behaviour).

plan_done.md

@@ -0,0 +1,143 @@
+# Similarity Report Feature — Work Done (Phases 0–4)
+
+This document records everything implemented for the **Corpus Similarity Report**
+feature on branch `report-ui-updated`, per `plan.md` + `opencode-prompt.md`.
+The feature is complete through Phase 4 (API). Phase 5 (PDF highlight mapper
+integration) was **not** started.
+
+> **Superseded 2026-08-02:** the remote merge described under "Remaining" is DONE
+> (merge commit `506ad88`). All 9 conflicts resolved; suite is 175 passed / 2 failed
+> (both `frontend-app/dist` env failures). See the post-merge box at the top of
+> `plan.md` for what the merge changed. Resume at Phase 5.
+
+All 5 commits are local-only (based on `33cb4a2`) and have **not** been pushed.
+The remote `report-ui-updated` branch has diverged (20+ commits of Groq-key UI,
+citation-relevance/novelty agents, frontend work, LF line endings) and contains
+**zero** similarity code.
+
+---
+
+## Commits
+
+| Commit | Description |
+|---|---|
+| `12cfcb6` | `feat(similarity): Phase 0 scoring core — normalize, exclusions, winnowing, matcher, aggregate` |
+| `69c39cd` | `feat(similarity): Phase 1 internal corpus — write path, read path, privacy` |
+| `658a727` | `feat(similarity): Phase 2 external providers — CORE, arXiv, OpenAlex, Crossref` |
+| `ecc8207` | `feat(similarity): Phase 3 pipeline + agent` |
+| `328d8c2` | `feat(similarity): Phase 4 API — report, recheck, status` |
+
+Working tree currently contains 212 files of pure CRLF-vs-LF noise
+(`git diff -w` collapses them to 0) plus one dirty LFS binary
+`models/kbir_semeval2017/training_args.bin`. Plan agreed with user:
+normalize line endings to LF in a housekeeping commit, then merge the remote.
+
+---
+
+## Files added
+
+### Similarity package (`similarity/`)
+- `__init__.py` — package export
+- `config.py` — settings + environment toggles
+- `schema.py` — `SimilarityReport`, `SourceMatch`, `Coverage`, providers
+- `normalize.py` — text normalization
+- `exclusions.py` — stopword/boilerplate exclusion
+- `fingerprint.py` — winnowing fingerprints (k=5, w=4)
+- `matcher.py` — matching + attribution
+- `aggregate.py` — aggregation into report
+- `index_writer.py` — corpus fingerprint write path
+- `selector.py` — internal-corpus read path
+- `pipeline.py` — `SimilarityPipeline` orchestration
+- `semantic.py` — semantic paraphrase pass
+- `corpus/base.py` — provider protocol
+- `corpus/internal.py` — internal corpus provider
+- `corpus/core_api.py` — CORE provider
+- `corpus/arxiv.py` — arXiv provider (https base)
+- `corpus/openalex.py` — OpenAlex provider
+- `corpus/crossref.py` — Crossref provider
+
+### Agent (`agents/similarity_agent.py`)
+`SimilarityAgent` (`name="similarity"`, `wave=3`, `critical=False`) — reads
+paragraph/section text from Neo4j, builds providers per `SIM_ENABLE_*` toggles,
+persists `SimilaritySource` nodes via `ctx.neo4j.run_write`.
+
+### Phase 4 API
+- `backend/routers/similarity.py` — `POST /similarity/{job_id}/recheck` (202,
+  409 in-flight guard, `BackgroundTasks`), `GET /similarity/status`
+- `backend/schemas/similarity.py` — `SimilarityStatus`, `RecheckAccepted`
+- `backend/tests/test_similarity.py` — 7 tests (report 200/202/404, recheck
+  accepted/409/404, status shape)
+
+## Files modified
+- `backend/routers/reports.py` — added `GET /reports/{job_id}/similarity`
+  (200 report / 202 not-ready / 404 unknown), `response_model=SimilarityReport`
+- `backend/main.py` — router registration
+- `orchestrators/custom_orchestrator.py` — `get_agents()` + similarity read-off,
+  `upsert_job`, `save_result` (3 spots)
+- `agents/__init__.py` — export
+- `configs/pipeline_config.yaml` — `similarity: wave 3, depends_on: [parser, keyword]`
+- `tests/test_agents.py` — +1 SimilarityAgent short-text/`unavailable` test
+- New tests: `tests/similarity/test_pipeline.py` (8), `test_self_match_guard.py`
+  (9), `test_semantic.py` (3)
+
+---
+
+## Key algorithm facts (stable)
+- Winnowing k=5, w=4; coverage bitmap `covered[i] = source_index|None`.
+- Attribution longest-match, ties by trust (internal > CORE > publisher).
+- Bucket percents computed independently; status `complete|partial|unavailable`
+  (< `SIM_MIN_DOCUMENT_WORDS`=300 → unavailable).
+- Headline is verbatim-only; semantic pass never touches `overall_percent`.
+- Preprint self-match guard: title token-overlap ≥ 0.9, DOI match, author overlap
+  ≥ 50% at similarity > 60%.
+- Budget timeout → `status="partial"` + `coverage.budget_exhausted=True`.
+- Provider exceptions caught → `last_error` + `coverage.providers_failed`.
+- Privacy boundary (plan §7.4): internal matches via `build_source_match()`
+  (neutral label, no URL/authors, stripped `source_excerpt`).
+- Recheck reuses stored Neo4j text (no PDF re-parse); `save_result` uses `$set`
+  merge so refresh is non-breaking.
+
+---
+
+## Test status (last full run)
+`tests/ backend/tests/ tests/similarity/` → **212 passed, 7 failed**. All 7
+failures are pre-existing and environment-related (not caused by the similarity
+work) — verified they also fail at Phase 3 commit `ecc8207`:
+
+- `tests/test_agents.py::test_scoring_agent_returns_result`
+- `tests/test_text_pipeline.py::test_no_detectors_enabled_returns_uncertain`
+- `backend/tests/test_health.py::test_root` (404 — no `frontend-app/dist`)
+- `backend/tests/test_reports.py::test_download_pdf` (500)
+- `backend/tests/test_text_detection.py` (3 tests — model returns REAL not
+  AI_GENERATED)
+
+`tests/similarity/` alone: 112/112 green. `backend/tests/test_similarity.py`: 7/7.
+
+Test command:
+```
+cd /mnt/c/citation_edge/IIT-Patna && timeout N /tmp/opencode/venv/bin/python -m pytest tests backend/tests tests/similarity -q
+```
+
+---
+
+## Environment notes
+- venv `/tmp/opencode/venv` (Py 3.12); missing torch/transformers installed
+  (CPU torch). Also installed python-dotenv, aiofiles, pdfplumber, reportlab,
+  pyyaml, tenacity, structlog, motor, pymongo, uvicorn, pymupdf, matplotlib,
+  scikit-learn, keybert, scipy, fastapi/alice httpx, pydantic 2.13.4.
+- Git auth: pull via askpass script `/tmp/opencode/git-askpass.sh`
+  (Username `x-access-token`), token supplied by user; never written to repo.
+- git user.name/user.email (per-repo): `Vansh Patil` / `vanshpatil@Vansh.localdomain`.
+- Commits carry a "Git LFS" hook warning (noise only).
+
+---
+
+## Remaining (Phase 5+)
+- Phase 5: PDF highlight mapper integration — combined AI+similarity span list,
+  `kind` field on spans, single mapper call (plan §8).
+- ~~Merge remote `report-ui-updated`~~ **DONE** (`506ad88`) — resolved 9 overlapping files
+  (`backend/main.py`, `backend/routers/reports.py`,
+  `orchestrators/custom_orchestrator.py`, `configs/pipeline_config.yaml`,
+  `agents/__init__.py`, `backend/routers/jobs.py`, `backend/tests/conftest.py`,
+  `backend/tests/test_jobs.py`, `tests/test_agents.py`).
+- ~~Push local commits~~ **DONE**.

services/academic_search.py

@@ -0,0 +1,217 @@
+"""Shared academic-search providers (OpenAlex, Crossref, arXiv).
+
+Lifted out of `services/paper_recommender.py` so both the citation-gap
+recommender and the similarity pipeline's corpus providers can import the
+same functions instead of copying them (plan.md §7.3 DRY note). `paper_recommender`
+re-imports from here, so its public surface is unchanged.
+"""
+
+import asyncio
+import re
+from typing import Dict, List
+from xml.etree import ElementTree
+
+import httpx
+
+from utils.logger import get_logger
+from utils.retry import async_retry
+
+logger = get_logger("academic_search")
+
+OPENALEX_BASE = "https://api.openalex.org/works"
+CROSSREF_BASE = "https://api.crossref.org/works"
+ARXIV_BASE = "https://export.arxiv.org/api/query"
+MAILTO = "citationedge@app.local"
+
+ARXIV_NS = {"atom": "http://www.w3.org/2005/Atom"}
+
+# DOI prefix → publisher (used to label where each paper comes from).
+DOI_PUBLISHER = {
+    "10.1016": "Elsevier",
+    "10.1007": "Springer",
+    "10.1002": "Wiley",
+    "10.1111": "Wiley",
+    "10.1093": "Oxford",
+    "10.1109": "IEEE",
+    "10.1038": "Nature",
+    "10.1021": "ACS",
+    "10.1126": "AAAS",
+    "10.1056": "NEJM",
+    "10.3390": "MDPI",
+    "10.1145": "ACM",
+    "10.1080": "Taylor & Francis",
+    "10.1175": "AMS",
+    "10.1088": "IOP",
+    "10.1063": "AIP",
+    "10.1017": "Cambridge",
+    "10.2139": "SSRN",
+    "10.3386": "NBER",
+    "10.48550": "arXiv",
+    "10.2469": "CFA Institute",
+    "10.3905": "Institutional Investor",
+    "10.2172": "OSTI",
+    "10.5281": "Zenodo",
+    "10.1257": "AEA",
+    "10.1371": "PLOS",
+    "10.18637": "JSS",
+    "10.1086": "University of Chicago Press",
+}
+
+# Shared throttle across all searches: providers are burst-sensitive
+# (OpenAlex in particular 429s on parallel bursts).
+PROVIDER_SEM = asyncio.Semaphore(2)
+
+_TAG_RE = re.compile(r"<[^>]+>")
+
+
+def clean_title(title: str) -> str:
+    """Strip HTML tags and unescape entities (OpenAlex titles carry <i>)."""
+    import html as _html
+    return _html.unescape(_TAG_RE.sub("", title or "")).strip()
+
+
+def _reconstruct_openalex_abstract(inverted: Dict | None) -> str:
+    if not inverted:
+        return ""
+    words: List[tuple[int, str]] = []
+    for word, positions in inverted.items():
+        words.extend((pos, word) for pos in positions)
+    words.sort()
+    return " ".join(w for _, w in words)
+
+
+def source_for(paper: Dict) -> str:
+    """Where is this paper published? DOI prefix → publisher → provider."""
+    doi = (paper.get("doi") or "").lower()
+    if doi.startswith("10."):
+        prefix = ".".join(doi.split(".")[:2])
+        return DOI_PUBLISHER.get(prefix, "Crossref")
+    url = (paper.get("url") or "").lower()
+    if "arxiv.org" in url:
+        return "arXiv"
+    if "europepmc.org" in url:
+        return "Europe PMC"
+    if "ncbi.nlm.nih.gov" in url:
+        return "PubMed"
+    if "openalex.org" in url:
+        return "OpenAlex"
+    if "crossref.org" in url or "doi.org" in url:
+        return "Crossref"
+    return "Web"
+
+
+@async_retry(max_attempts=3, delay=1.5)
+async def _search_openalex(client: httpx.AsyncClient, query: str, limit: int = 10) -> List[Dict]:
+    resp = await client.get(
+        OPENALEX_BASE,
+        params={"search": query, "per-page": limit, "mailto": MAILTO},
+        timeout=20,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    results = []
+    for w in data.get("results", []):
+        display = (w.get("display_name") or "").strip()
+        if not display:
+            continue
+        doi = (w.get("doi") or "").replace("https://doi.org/", "")
+        results.append({
+            "title": display,
+            "authors": [
+                a.get("author", {}).get("display_name", "")
+                for a in w.get("authorships", [])[:5]
+            ],
+            "year": w.get("publication_year"),
+            "abstract": _reconstruct_openalex_abstract(w.get("abstract_inverted_index")),
+            "citation_count": w.get("cited_by_count", 0),
+            "doi": doi,
+            "paper_id": w.get("id", ""),
+            "url": w.get("id", ""),  # landing page: https://openalex.org/W...
+        })
+    return results
+
+
+CROSSREF_TYPES = {
+    "journal-article", "proceedings-article", "posted-content",
+    "preprint", "report", "dissertation",
+}
+
+
+@async_retry(max_attempts=3, delay=1.5)
+async def _search_crossref(client: httpx.AsyncClient, query: str, limit: int = 10) -> List[Dict]:
+    resp = await client.get(
+        CROSSREF_BASE,
+        params={"query": query, "rows": limit, "select": "title,author,issued,DOI,is-referenced-by-count,link,URL,type,abstract"},
+        timeout=20,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    results = []
+    for item in data.get("message", {}).get("items", []):
+        if item.get("type") not in CROSSREF_TYPES:
+            continue
+        title = (item.get("title") or [""])[0].strip()
+        if not title:
+            continue
+        doi = item.get("DOI", "")
+        issued = (item.get("issued", {}).get("date-parts") or [[None]])[0][0]
+        link = item.get("URL") or item.get("link", [{}])[0].get("URL", "")
+        if not link and doi:
+            link = f"https://doi.org/{doi}"
+        abstract = (item.get("abstract") or "")
+        abstract = clean_title(abstract)[:500]
+        results.append({
+            "title": title,
+            "authors": [a.get("given", "") + " " + a.get("family", "") for a in item.get("author", [])[:5]],
+            "year": issued,
+            "abstract": abstract,
+            "citation_count": item.get("is-referenced-by-count", 0),
+            "doi": doi,
+            "paper_id": doi,
+            "url": link,
+        })
+    return results
+
+
+@async_retry(max_attempts=3, delay=1.5)
+async def _search_arxiv(client: httpx.AsyncClient, query: str, limit: int = 10) -> List[Dict]:
+    """arXiv API — direct arxiv.org/abs/... links (Atom XML)."""
+    resp = await client.get(
+        ARXIV_BASE,
+        params={"search_query": f"all:{query}", "max_results": limit, "sortBy": "relevance"},
+        timeout=20,
+    )
+    resp.raise_for_status()
+    root = ElementTree.fromstring(resp.text)
+    results = []
+    for entry in root.findall("atom:entry", ARXIV_NS):
+        title = clean_title(entry.findtext("atom:title", "", ARXIV_NS))
+        if not title:
+            continue
+        arxiv_id = entry.findtext("atom:id", "", ARXIV_NS).strip()
+        if not arxiv_id:
+            continue
+        doi = next(
+            (el.text or "").strip() for el in entry.iter()
+            if el.tag.rsplit("}", 1)[-1] == "doi" and el.text
+        ) if any(el.tag.rsplit("}", 1)[-1] == "doi" and el.text for el in entry.iter()) else ""
+        year = None
+        published = entry.findtext("atom:published", "", ARXIV_NS)
+        if published:
+            year = int(published[:4])
+        authors = [
+            a.findtext("atom:name", "", ARXIV_NS).strip()
+            for a in entry.findall("atom:author", ARXIV_NS)[:5]
+        ]
+        summary = " ".join(entry.findtext("atom:summary", "", ARXIV_NS).split())
+        results.append({
+            "title": title,
+            "authors": [a for a in authors if a],
+            "year": year,
+            "abstract": summary,
+            "citation_count": 0,  # arXiv has no citation count
+            "doi": doi,
+            "paper_id": arxiv_id,
+            "url": arxiv_id,  # direct link: https://arxiv.org/abs/XXXX
+        })
+    return results

services/paper_recommender.py

@@ -11,62 +11,35 @@ Providers:
   1. OpenAlex   — broad aggregator (publishers, repositories)
   2. Crossref   — publisher metadata (Elsevier, Springer, IEEE, Wiley, ...)
   3. arXiv      — open-access preprint repository (arxiv.org direct links)
-  4. Europe PMC — biomedical literature + preprints + patents (keyless)
-  5. PubMed     — MEDLINE abstracts via NCBI E-utilities (keyless)
+  4. Europe PMC — open-access life-science literature
+  5. PubMed     — biomedical literature (E-utilities)
+  All keyless; Unpaywall adds open-access links to the top results.
+
+The provider functions themselves live in `services/academic_search.py`
+(shared with the similarity corpus providers - see plan.md §7.3) and are
+re-imported here so the public surface of this module is unchanged.
 """
 import asyncio
 import math
 import re
 from typing import Dict, List, Optional, Tuple
-from xml.etree import ElementTree
 
 import httpx
 
+from services.academic_search import (
+    DOI_PUBLISHER,
+    PROVIDER_SEM,
+    _search_arxiv,
+    _search_crossref,
+    _search_openalex,
+    clean_title,
+    source_for,
+)
 from tools.search_tool import EuropePMCTool, PubMedTool, UnpaywallTool
 from utils.logger import get_logger
-from utils.retry import async_retry
 
 logger = get_logger("paper_recommender")
 
-OPENALEX_BASE = "https://api.openalex.org/works"
-CROSSREF_BASE = "https://api.crossref.org/works"
-ARXIV_BASE = "http://export.arxiv.org/api/query"
-MAILTO = "citationedge@app.local"
-
-ARXIV_NS = {"atom": "http://www.w3.org/2005/Atom"}
-
-# DOI prefix → publisher (used to label where each paper comes from).
-DOI_PUBLISHER = {
-    "10.1016": "Elsevier",
-    "10.1007": "Springer",
-    "10.1002": "Wiley",
-    "10.1111": "Wiley",
-    "10.1093": "Oxford",
-    "10.1109": "IEEE",
-    "10.1038": "Nature",
-    "10.1021": "ACS",
-    "10.1126": "AAAS",
-    "10.1056": "NEJM",
-    "10.3390": "MDPI",
-    "10.1145": "ACM",
-    "10.1080": "Taylor & Francis",
-    "10.1175": "AMS",
-    "10.1088": "IOP",
-    "10.1063": "AIP",
-    "10.1017": "Cambridge",
-    "10.2139": "SSRN",
-    "10.3386": "NBER",
-    "10.48550": "arXiv",
-    "10.2469": "CFA Institute",
-    "10.3905": "Institutional Investor",
-    "10.2172": "OSTI",
-    "10.5281": "Zenodo",
-    "10.1257": "AEA",
-    "10.1371": "PLOS",
-    "10.18637": "JSS",
-    "10.1086": "University of Chicago Press",
-}
-
 STOPWORDS = {
     "a", "an", "the", "of", "on", "in", "to", "for", "and", "or", "is", "are",
     "was", "were", "be", "been", "being", "it", "its", "this", "that", "these",
@@ -116,7 +89,6 @@ FILLER_TOKENS = {
 }
 
 _TOKEN_RE = re.compile(r"[a-z0-9]+")
-_TAG_RE = re.compile(r"<[^>]+>")
 
 # Tokens too weak/generic to help search queries ("for example", "even",
 # "using"...) plus pure-numeric noise like "1m", "2m".
@@ -129,12 +101,6 @@ WEAK_QUERY_TOKENS = {
 }
 
 
-def clean_title(title: str) -> str:
-    """Strip HTML tags and unescape entities (OpenAlex titles carry <i>)."""
-    import html as _html
-    return _html.unescape(_TAG_RE.sub("", title or "")).strip()
-
-
 def tokenize(text: str) -> List[str]:
     """Gap/query tokens: stopwords + gap-phrasing fillers removed, deduped."""
     if not text:
@@ -299,10 +265,6 @@ def rank_papers(
 
 # ── Providers ────────────────────────────────────────────────────────────────
 
-# Shared throttle across all searches: providers are burst-sensitive
-# (OpenAlex in particular 429s on parallel bursts).
-_PROVIDER_SEM = asyncio.Semaphore(2)
-
 
 async def _search_all_providers(
     client: httpx.AsyncClient, query: str, limit: int = 10
@@ -312,7 +274,7 @@ async def _search_all_providers(
     results: List[Dict] = []
     epmc = EuropePMCTool()
     pubmed = PubMedTool()
-    async with _PROVIDER_SEM:
+    async with PROVIDER_SEM:
         for fn in (_search_openalex, _search_crossref, _search_arxiv):
             try:
                 results.extend(await fn(client, query, limit))
@@ -327,152 +289,6 @@ async def _search_all_providers(
             await asyncio.sleep(0.25)
     return results
 
-def _reconstruct_openalex_abstract(inverted: Optional[Dict]) -> str:
-    if not inverted:
-        return ""
-    words: List[Tuple[int, str]] = []
-    for word, positions in inverted.items():
-        words.extend((pos, word) for pos in positions)
-    words.sort()
-    return " ".join(w for _, w in words)
-
-
-@async_retry(max_attempts=3, delay=1.5)
-async def _search_openalex(client: httpx.AsyncClient, query: str, limit: int = 10) -> List[Dict]:
-    resp = await client.get(
-        OPENALEX_BASE,
-        params={"search": query, "per-page": limit, "mailto": MAILTO},
-        timeout=20,
-    )
-    resp.raise_for_status()
-    data = resp.json()
-    results = []
-    for w in data.get("results", []):
-        display = (w.get("display_name") or "").strip()
-        if not display:
-            continue
-        doi = (w.get("doi") or "").replace("https://doi.org/", "")
-        results.append({
-            "title": display,
-            "authors": [
-                a.get("author", {}).get("display_name", "")
-                for a in w.get("authorships", [])[:5]
-            ],
-            "year": w.get("publication_year"),
-            "abstract": _reconstruct_openalex_abstract(w.get("abstract_inverted_index")),
-            "citation_count": w.get("cited_by_count", 0),
-            "doi": doi,
-            "paper_id": w.get("id", ""),
-            "url": w.get("id", ""),  # landing page: https://openalex.org/W...
-        })
-    return results
-
-
-CROSSREF_TYPES = {
-    "journal-article", "proceedings-article", "posted-content",
-    "preprint", "report", "dissertation",
-}
-
-
-@async_retry(max_attempts=3, delay=1.5)
-async def _search_crossref(client: httpx.AsyncClient, query: str, limit: int = 10) -> List[Dict]:
-    resp = await client.get(
-        CROSSREF_BASE,
-        params={"query": query, "rows": limit, "select": "title,author,issued,DOI,is-referenced-by-count,link,URL,type,abstract"},
-        timeout=20,
-    )
-    resp.raise_for_status()
-    data = resp.json()
-    results = []
-    for item in data.get("message", {}).get("items", []):
-        if item.get("type") not in CROSSREF_TYPES:
-            continue
-        title = (item.get("title") or [""])[0].strip()
-        if not title:
-            continue
-        doi = item.get("DOI", "")
-        issued = (item.get("issued", {}).get("date-parts") or [[None]])[0][0]
-        link = item.get("URL") or item.get("link", [{}])[0].get("URL", "")
-        if not link and doi:
-            link = f"https://doi.org/{doi}"
-        abstract = (item.get("abstract") or "")
-        abstract = clean_title(abstract)[:500]
-        results.append({
-            "title": title,
-            "authors": [a.get("given", "") + " " + a.get("family", "") for a in item.get("author", [])[:5]],
-            "year": issued,
-            "abstract": abstract,
-            "citation_count": item.get("is-referenced-by-count", 0),
-            "doi": doi,
-            "paper_id": doi,
-            "url": link,
-        })
-    return results
-
-
-def source_for(paper: Dict) -> str:
-    """Where is this paper published? DOI prefix → publisher → provider."""
-    doi = (paper.get("doi") or "").lower()
-    if doi.startswith("10."):
-        prefix = ".".join(doi.split(".")[:2])
-        return DOI_PUBLISHER.get(prefix, "Crossref")
-    url = (paper.get("url") or "").lower()
-    if "arxiv.org" in url:
-        return "arXiv"
-    if "europepmc.org" in url or "pmc.ncbi.nlm.nih.gov" in url:
-        return "Europe PMC"
-    if "pubmed.ncbi.nlm.nih.gov" in url:
-        return "PubMed"
-    if "openalex.org" in url:
-        return "OpenAlex"
-    if "crossref.org" in url or "doi.org" in url:
-        return "Crossref"
-    return "Web"
-
-
-@async_retry(max_attempts=3, delay=1.5)
-async def _search_arxiv(client: httpx.AsyncClient, query: str, limit: int = 10) -> List[Dict]:
-    """arXiv API — direct arxiv.org/abs/... links (Atom XML)."""
-    resp = await client.get(
-        ARXIV_BASE,
-        params={"search_query": f"all:{query}", "max_results": limit, "sortBy": "relevance"},
-        timeout=20,
-    )
-    resp.raise_for_status()
-    root = ElementTree.fromstring(resp.text)
-    results = []
-    for entry in root.findall("atom:entry", ARXIV_NS):
-        title = clean_title(entry.findtext("atom:title", "", ARXIV_NS))
-        if not title:
-            continue
-        arxiv_id = entry.findtext("atom:id", "", ARXIV_NS).strip()
-        if not arxiv_id:
-            continue
-        doi = next(
-            (el.text or "").strip() for el in entry.iter()
-            if el.tag.rsplit("}", 1)[-1] == "doi" and el.text
-        ) if any(el.tag.rsplit("}", 1)[-1] == "doi" and el.text for el in entry.iter()) else ""
-        year = None
-        published = entry.findtext("atom:published", "", ARXIV_NS)
-        if published:
-            year = int(published[:4])
-        authors = [
-            a.findtext("atom:name", "", ARXIV_NS).strip()
-            for a in entry.findall("atom:author", ARXIV_NS)[:5]
-        ]
-        summary = " ".join(entry.findtext("atom:summary", "", ARXIV_NS).split())
-        results.append({
-            "title": title,
-            "authors": [a for a in authors if a],
-            "year": year,
-            "abstract": summary,
-            "citation_count": 0,  # arXiv has no citation count
-            "doi": doi,
-            "paper_id": arxiv_id,
-            "url": arxiv_id,  # direct link: https://arxiv.org/abs/XXXX
-        })
-    return results
-
 
 class PaperRecommender:
     """Web-based paper recommender (OpenAlex + Crossref + arXiv + Europe PMC
@@ -555,16 +371,15 @@ class PaperRecommender:
                 chosen = {r["title"].lower() for r in recs}
                 recs += [r for r in filled if r["title"].lower() not in chosen]
 
-        # Enrich top recommendations with open-access copies (silent on failure)
+        # Unpaywall enrichment: attach a direct open-access URL where possible.
         for rec in recs:
-            if not rec.get("doi"):
-                continue
-            try:
-                result = await self._unpaywall.run(rec["doi"])
-                if result.success and result.data and result.data.get("url"):
-                    rec["oa_url"] = result.data["url"]
-            except Exception as e:
-                logger.debug(f"Unpaywall enrichment failed for {rec['doi']}: {e}")
+            if rec.get("doi"):
+                try:
+                    result = await self._unpaywall.run(rec["doi"])
+                    if result.success and result.data and result.data.get("url"):
+                        rec["oa_url"] = result.data["url"]
+                except Exception as e:
+                    logger.debug(f"Unpaywall enrichment failed for {rec['doi']}: {e}")
 
         return recs[:limit]
 
@@ -579,9 +394,8 @@ class PaperRecommender:
         keywords: Optional[List[str]] = None,
     ) -> List[Dict]:
         """
-        Raw academic search (OpenAlex + Crossref + arXiv + Europe PMC +
-        PubMed) without ranking: returns candidates labeled with source,
-        for evidence gathering.
+        Raw academic search (OpenAlex + Crossref + arXiv) without ranking:
+        returns candidates labeled with source, for evidence gathering.
         """
         query_terms = extract_query(query, keywords)
         q = " ".join(query_terms[:4]) if query_terms else query.strip()

similarity/aggregate.py

@@ -0,0 +1,98 @@
+"""Aggregation: the coverage bitmap and every percentage in the report.
+
+The single data structure the whole score rests on (plan.md §4.2):
+
+    covered[i] = source_index that owns word i, or None
+
+A word is counted once no matter how many sources contain it - this is what
+stops percentages exceeding 100% when boilerplate appears in forty papers.
+Attribution (§4.3) goes to the source with the longest confirmed match
+overlapping that word; ties break on provider trust. Bucket percentages (§4.4)
+are computed independently, as "the index if this were the only bucket" - they
+deliberately do not sum to the headline number.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .schema import Bucket
+
+
+@dataclass
+class Match:
+    """One confirmed overlap, ready for the bitmap."""
+
+    source_index: int
+    bucket: Bucket
+    trust: int          # higher preferred when lengths tie
+    doc_start: int      # inclusive
+    doc_end: int        # exclusive
+
+
+@dataclass
+class ScoreResult:
+    covered: list[int | None]                 # per-word owning source index
+    overall_percent: int
+    per_source: dict[int, int]                # source_index -> attributed words
+    per_source_percent: dict[int, int]        # source_index -> percent
+    bucket_words: dict[Bucket, int]           # words covered by ANY source in bucket
+    bucket_percents: dict[Bucket, int]        # bucket "as if it were the only one"
+
+
+def _pct(count: int, total: int) -> int:
+    if total <= 0:
+        return 0
+    return round(100 * count / total)
+
+
+def compute_percentages(
+    total_words: int,
+    matches: list[Match],
+    min_match_words: int = 8,
+) -> ScoreResult:
+    """Fill the coverage bitmap and derive all four percentage families."""
+    kept = [m for m in matches if m.doc_end - m.doc_start >= min_match_words]
+
+    covered: list[int | None] = [None] * total_words
+    best_len = [0] * total_words
+    best_trust = [0] * total_words
+
+    for m in kept:
+        length = m.doc_end - m.doc_start
+        for i in range(m.doc_start, min(m.doc_end, total_words)):
+            if covered[i] is None or (length, m.trust) > (best_len[i], best_trust[i]):
+                covered[i] = m.source_index
+                best_len[i] = length
+                best_trust[i] = m.trust
+
+    covered_count = sum(1 for c in covered if c is not None)
+
+    per_source: dict[int, int] = {}
+    for c in covered:
+        if c is not None:
+            per_source[c] = per_source.get(c, 0) + 1
+    per_source_percent = {s: _pct(n, total_words) for s, n in per_source.items()}
+
+    # Only buckets that actually contributed matches are reported; a bucket
+    # with no confirmed match must not claim it was "checked and clean".
+    bucket_words: dict[Bucket, int] = {}
+    bucket_percents: dict[Bucket, int] = {}
+    for bucket in {m.bucket for m in kept}:
+        seen: set[int] = set()
+        for m in kept:
+            if m.bucket is not bucket:
+                continue
+            for i in range(m.doc_start, min(m.doc_end, total_words)):
+                seen.add(i)
+        bucket_words[bucket] = len(seen)
+        bucket_percents[bucket] = _pct(len(seen), total_words)
+
+    return ScoreResult(
+        covered=covered,
+        overall_percent=_pct(covered_count, total_words),
+        per_source=per_source,
+        per_source_percent=per_source_percent,
+        bucket_words=bucket_words,
+        bucket_percents=bucket_percents,
+    )

similarity/config.py

@@ -0,0 +1,57 @@
+"""Settings for the corpus-similarity package.
+
+Namespaced under the SIM_ env prefix so it cannot collide with CitationEdge's
+own configuration, and `extra="ignore"` so the shared .env (Groq/Neo4j/Mongo
+keys) is read without complaint. The similarity pipeline ships ENABLED but its
+corpus indexing ships OFF: indexing users' papers into a shared corpus is a
+consent decision, not a default (see plan.md §16 risk 6).
+"""
+
+from __future__ import annotations
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class SimilaritySettings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="SIM_",
+        env_file=".env",
+        extra="ignore",
+    )
+
+    # -- Master switch ---------------------------------------------
+    enabled: bool = True
+    core_api_key: str = ""   # CORE v3 key; empty => unauthenticated (rate-limited)
+
+    # -- Per-provider toggles --------------------------------------
+    enable_core: bool = True
+    enable_arxiv: bool = True
+    enable_openalex: bool = True
+    enable_crossref: bool = True
+    enable_internal: bool = True
+    # Index this document's fingerprints into the internal corpus so it can be
+    # matched against later submissions. SHIPS OFF - a consent decision.
+    index_uploads: bool = False
+
+    # -- Fingerprinting --------------------------------------------
+    kgram_size: int = 5          # winnowing k, in words
+    window_size: int = 4         # winnowing w; detects any shared run >= w+k-1
+    min_match_words: int = 8     # minimum reportable match length
+    min_document_words: int = 300  # below this => unavailable
+
+    # -- Retrieval budget ------------------------------------------
+    max_queries: int = 24        # phrase queries per document
+    max_candidates: int = 400    # candidates verified per document
+    max_pdf_fetches: int = 5     # OA PDFs downloaded per document
+    budget_seconds: int = 45     # hard wall-clock cap on retrieval
+
+    # -- Exclusions ------------------------------------------------
+    exclude_quotes: bool = True
+    exclude_bibliography: bool = True
+
+    # -- Paraphrase signal (never in the headline %) ---------------
+    paraphrase_threshold: float = 0.92
+    enable_paraphrase: bool = True
+
+
+settings = SimilaritySettings()

similarity/corpus/arxiv.py

@@ -0,0 +1,120 @@
+"""arXiv provider (plan.md §7.2).
+
+Metadata comes from the shared `_search_arxiv` (reused verbatim from
+`services/academic_search.py`, itself lifted from paper_recommender - never
+copied). Full text is fetched by downloading the OA PDF and extracting it
+with the existing `utils.pdf.extract_text_from_pdf`, capped at
+`SIM_MAX_PDF_FETCHES`. PDF fetch + parse is the slowest thing in the
+pipeline, so the cap and the shared semaphore matter.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import tempfile
+from pathlib import Path
+
+import httpx
+
+from similarity.config import settings
+from similarity.corpus.base import Candidate
+from similarity.schema import Bucket
+from utils.logger import get_logger
+from utils.retry import async_retry
+
+logger = get_logger("arxiv_provider")
+
+# Shared throttle across all providers (plan.md §7.1).
+_ARXIV_SEM = asyncio.Semaphore(2)
+
+
+class ArxivProvider:
+    """Search arXiv metadata; fetch full text from the OA PDF when asked."""
+
+    name = "arxiv"
+    provider = "arXiv"
+    bucket = Bucket.OPEN_ACCESS
+
+    def __init__(
+        self,
+        client: httpx.AsyncClient | None = None,
+        max_pdf_fetches: int | None = None,
+        max_attempts: int = 3,
+        delay: float = 1.0,
+        backoff: float = 2.0,
+    ):
+        self._client = client or httpx.AsyncClient()
+        self._fetches_left = max_pdf_fetches if max_pdf_fetches is not None else settings.max_pdf_fetches
+        self.last_error: str | None = None
+        self._search_once = async_retry(
+            max_attempts=max_attempts, delay=delay, backoff=backoff,
+            exceptions=(httpx.HTTPStatusError, httpx.TransportError),
+        )(self._search_once_raw)
+
+    async def search_phrase(self, phrase: str, limit: int) -> list[Candidate]:
+        try:
+            results = await self._search_once(phrase, limit)
+        except Exception as e:
+            self.last_error = str(e)
+            logger.warning(f"arXiv search failed for {phrase[:40]!r}: {e}")
+            return []
+        candidates: list[Candidate] = []
+        for r in results:
+            url = (r.get("url") or "").strip()
+            if not url:
+                continue
+            candidates.append(
+                Candidate(
+                    candidate_id=r.get("paper_id") or url,
+                    title=r.get("title", ""),
+                    url=url,
+                    doi=r.get("doi") or None,
+                    authors=list(r.get("authors") or []),
+                    year=r.get("year"),
+                    provider=self.provider,
+                    bucket=self.bucket,
+                    text=r.get("abstract") or None,
+                )
+            )
+        return candidates[:limit]
+
+    async def fetch_text(self, candidate: Candidate) -> str | None:
+        """Download the OA PDF and extract its text; budget-capped."""
+        if self._fetches_left <= 0:
+            logger.warning("arXiv PDF fetch budget exhausted")
+            return None
+        pdf_url = candidate.url.replace("/abs/", "/pdf/")
+        if not pdf_url.endswith(".pdf"):
+            pdf_url = f"{pdf_url}.pdf"
+
+        try:
+            text = await self._download_and_extract(pdf_url)
+        except Exception as e:  # noqa: BLE001 - a bad PDF must not kill the run
+            self.last_error = str(e)
+            logger.warning(f"arXiv PDF fetch failed for {candidate.url}: {e}")
+            return None
+
+        self._fetches_left -= 1
+        return text
+
+    async def _download_and_extract(self, pdf_url: str) -> str:
+        async with _ARXIV_SEM:
+            await asyncio.sleep(0.25)
+            resp = await self._client.get(pdf_url, timeout=30, follow_redirects=True)
+            resp.raise_for_status()
+        with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
+            tmp.write(resp.content)
+            tmp_path = tmp.name
+        try:
+            from utils.pdf import extract_text_from_pdf
+            result = extract_text_from_pdf(tmp_path)
+        finally:
+            Path(tmp_path).unlink(missing_ok=True)
+        full_text = result.get("full_text") or ""
+        return full_text if full_text.strip() else None
+
+    async def _search_once_raw(self, phrase: str, limit: int) -> list[dict]:
+        from services.academic_search import _search_arxiv
+        async with _ARXIV_SEM:
+            await asyncio.sleep(0.25)
+            return await _search_arxiv(self._client, phrase, limit)

similarity/corpus/base.py

@@ -0,0 +1,43 @@
+"""Corpus provider protocol and the candidate record every provider returns.
+
+Plan.md §7: all providers (CORE, arXiv, OpenAlex, Crossref, internal corpus)
+implement one interface so the pipeline is provider-agnostic and each can be
+switched off independently. A `Candidate` is the *proposal* stage of the
+two-stage design - stage two (verification against real text) lives in
+`similarity.matcher`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Protocol
+
+from similarity.schema import Bucket
+
+
+@dataclass
+class Candidate:
+    """A document worth verifying, returned by a provider's phrase search."""
+
+    source_index: int | None = None  # assigned by the pipeline once ranked
+    candidate_id: str = ""
+    title: str = ""
+    url: str = ""                    # must be resolvable before it can become evidence
+    doi: str | None = None
+    authors: list[str] = field(default_factory=list)
+    year: int | None = None
+    provider: str = ""               # "CORE" | "arXiv" | "Crossref" | "OpenAlex" | "CitationEdge Corpus"
+    bucket: Bucket | None = None
+    text: str | None = None          # pre-fetched full text (CORE returns it inline)
+    submitted_at: str = ""           # internal corpus: neutral "submitted YYYY-MM-DD" label
+
+
+class CorpusProvider(Protocol):
+    """Anything that can propose candidates for a phrase and fetch their text."""
+
+    name: str
+    bucket: Bucket
+
+    async def search_phrase(self, phrase: str, limit: int) -> list[Candidate]: ...
+
+    async def fetch_text(self, candidate: Candidate) -> str | None: ...

similarity/corpus/core_api.py

@@ -0,0 +1,108 @@
+"""CORE v3 — the full-text workhorse (plan.md §7.1).
+
+`GET https://api.core.ac.uk/v3/search/works?q=fullText:"<phrase>"` returns
+records that can carry the FULL TEXT inline, so retrieval and text fetch
+collapse into one round trip. The unauthenticated limit is ~5 requests /
+10 seconds, which 24 phrase queries would blow through immediately - hence
+the startup warning when `SIM_CORE_API_KEY` is unset, the shared semaphore,
+and the stagger between calls.
+
+Failures are surfaced, never swallowed: after `@async_retry` exhausts, the
+provider records `last_error` and returns `[]`, so the pipeline can append the
+provider name to `coverage.providers_failed` (plan.md §16 risk 4).
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+
+from similarity.config import settings
+from similarity.corpus.base import Candidate
+from similarity.schema import Bucket
+from utils.logger import get_logger
+from utils.retry import async_retry
+
+logger = get_logger("core_api")
+
+CORE_SEARCH_URL = "https://api.core.ac.uk/v3/search/works"
+
+# Shared throttle across ALL providers (plan.md §7.1): bursts 429 instantly.
+_CORE_SEM = asyncio.Semaphore(2)
+
+if not settings.core_api_key:
+    logger.warning(
+        "SIM_CORE_API_KEY is unset: CORE will run unauthenticated "
+        "(~5 requests / 10s) and rate-limit immediately. Register at "
+        "core.ac.uk/docs/ to get a key."
+    )
+
+
+class CoreApiProvider:
+    """Full-text search + fetch from CORE, one round trip per phrase."""
+
+    name = "core"
+    provider = "CORE"
+    bucket = Bucket.OPEN_ACCESS
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        client: httpx.AsyncClient | None = None,
+        max_attempts: int = 3,
+        delay: float = 1.0,
+        backoff: float = 2.0,
+    ):
+        self._api_key = (api_key if api_key is not None else settings.core_api_key) or ""
+        self._client = client or httpx.AsyncClient()
+        self.last_error: str | None = None
+        self._search_once = async_retry(
+            max_attempts=max_attempts, delay=delay, backoff=backoff,
+            exceptions=(httpx.HTTPStatusError, httpx.TransportError),
+        )(self._search_once_raw)
+
+    async def search_phrase(self, phrase: str, limit: int) -> list[Candidate]:
+        try:
+            data = await self._search_once(phrase, limit)
+            results = data.get("results", []) if isinstance(data, dict) else []
+            return [self._to_candidate(r) for r in results[:limit]]
+        except Exception as e:
+            self.last_error = str(e)
+            logger.warning(f"CORE search failed for {phrase[:40]!r}: {e}")
+            return []
+
+    async def fetch_text(self, candidate: Candidate) -> str | None:
+        """CORE returns the full text inline at search time."""
+        return candidate.text
+
+    async def _search_once_raw(self, phrase: str, limit: int) -> dict:
+        headers = {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {}
+        params = {"q": f'fullText:"{phrase}"', "limit": limit, "scroll": False}
+        async with _CORE_SEM:
+            await asyncio.sleep(0.25)  # stagger, kind to rate limits
+            resp = await self._client.get(
+                CORE_SEARCH_URL, params=params, headers=headers, timeout=20
+            )
+            resp.raise_for_status()
+            return resp.json()
+
+    def _to_candidate(self, record: dict) -> Candidate:
+        title = (record.get("title") or "").strip()
+        url = (record.get("downloadUrl") or record.get("fullTextUrl") or "").strip()
+        if not url:
+            # Metadata-only records carry no download URL; fall back to DOI.
+            doi = (record.get("doi") or "").strip()
+            if doi:
+                url = f"https://doi.org/{doi}"
+        return Candidate(
+            candidate_id=(record.get("id") or url or title),
+            title=title,
+            url=url,
+            doi=(record.get("doi") or "").strip() or None,
+            authors=[a for a in (record.get("authors") or []) if a][:10],
+            year=record.get("yearPublished"),
+            provider=self.provider,
+            bucket=self.bucket,
+            text=record.get("fullText") or record.get("abstract") or None,
+        )

similarity/corpus/crossref.py

@@ -0,0 +1,84 @@
+"""Crossref provider (plan.md §7.3).
+
+Abstract-only matching, reusing the shared `_search_crossref` verbatim. Like
+OpenAlex these are mostly 1% PUBLICATION-bucket entries; the DOI prefix map
+(`DOI_PUBLISHER`) labels them as Elsevier / Springer / IEEE / ... exactly as
+the reference screenshot shows.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+
+from services.academic_search import DOI_PUBLISHER, _search_crossref, source_for
+from similarity.corpus.base import Candidate
+from similarity.schema import Bucket
+from utils.logger import get_logger
+from utils.retry import async_retry
+
+logger = get_logger("crossref_provider")
+
+# Shared throttle across all providers (plan.md §7.1).
+_CROSSREF_SEM = asyncio.Semaphore(2)
+
+
+class CrossrefProvider:
+    """Search Crossref metadata; the abstract is the only retrievable text."""
+
+    name = "crossref"
+    provider = "Crossref"
+    bucket = Bucket.PUBLICATION
+
+    def __init__(
+        self,
+        client: httpx.AsyncClient | None = None,
+        max_attempts: int = 3,
+        delay: float = 1.0,
+        backoff: float = 2.0,
+    ):
+        self._client = client or httpx.AsyncClient()
+        self.last_error: str | None = None
+        self._search_once = async_retry(
+            max_attempts=max_attempts, delay=delay, backoff=backoff,
+            exceptions=(httpx.HTTPStatusError, httpx.TransportError),
+        )(self._search_once_raw)
+
+    async def search_phrase(self, phrase: str, limit: int) -> list[Candidate]:
+        try:
+            results = await self._search_once(phrase, limit)
+        except Exception as e:
+            self.last_error = str(e)
+            logger.warning(f"Crossref search failed for {phrase[:40]!r}: {e}")
+            return []
+        candidates: list[Candidate] = []
+        for r in results:
+            url = (r.get("url") or "").strip()
+            if not url:
+                continue
+            doi = r.get("doi") or None
+            label = DOI_PUBLISHER.get((doi or "").split(".")[0] + "." + (doi or "").split(".")[1], "Crossref") if doi and doi.startswith("10.") else source_for(r)
+            candidates.append(
+                Candidate(
+                    candidate_id=r.get("paper_id") or url,
+                    title=r.get("title", ""),
+                    url=url,
+                    doi=doi,
+                    authors=list(r.get("authors") or []),
+                    year=r.get("year"),
+                    provider=self.provider,
+                    bucket=self.bucket,
+                    text=r.get("abstract") or None,
+                )
+            )
+        return candidates[:limit]
+
+    async def fetch_text(self, candidate: Candidate) -> str | None:
+        """Crossref serves abstracts only - there is no full text to fetch."""
+        return candidate.text
+
+    async def _search_once_raw(self, phrase: str, limit: int) -> list[dict]:
+        async with _CROSSREF_SEM:
+            await asyncio.sleep(0.25)
+            return await _search_crossref(self._client, phrase, limit)

similarity/corpus/internal.py

@@ -0,0 +1,112 @@
+"""The internal corpus provider: prior CitationEdge uploads (plan.md §7.4).
+
+Read path: fingerprint the query phrase, ask the Mongo multikey index for any
+stored document sharing a fingerprint, then hand back the raw text so
+`similarity.matcher` can verify locally. The winnowing guarantee transfers
+verbatim: if the phrase (>= w+k-1 words) appears inside a stored document,
+the two fingerprint sets intersect, so the candidate is found.
+
+Privacy is enforced HERE, at construction, not in the UI where a future
+refactor could silently drop it: `SourceMatch.title` and `.url` are replaced
+with a neutral submission-date label, `authors` is emptied, and the
+`source_excerpt` (text belonging to the *other* researcher's paper) is
+stripped from every span.
+"""
+
+from __future__ import annotations
+
+from similarity.config import settings
+from similarity.corpus.base import Candidate
+from similarity.fingerprint import fingerprint
+from similarity.normalize import normalize
+from similarity.schema import Bucket, SourceMatch
+
+
+def _submitted_label(created_at) -> str:
+    """'CitationEdge Corpus · submitted YYYY-MM-DD' from a stored row."""
+    if hasattr(created_at, "date"):
+        date = created_at.date().isoformat()
+    else:
+        date = str(created_at or "")[:10] or "unknown-date"
+    return f"CitationEdge Corpus · submitted {date}"
+
+
+class InternalCorpusProvider:
+    """Search prior uploads and verify matches against their stored text."""
+
+    name = "internal"
+    bucket = Bucket.INTERNAL
+    provider = "CitationEdge Corpus"
+
+    def __init__(self, mongo, *, doc_id: str, job_id: str | None = None,
+                 k: int | None = None, w: int | None = None):
+        self._mongo = mongo
+        self._doc_id = doc_id
+        self._job_id = job_id
+        self._k = k or settings.kgram_size
+        self._w = w or settings.window_size
+
+    async def search_phrase(self, phrase: str, limit: int) -> list[Candidate]:
+        if not settings.enable_internal:
+            return []
+        nd = normalize(phrase)
+        fps = fingerprint(nd.words, self._k, self._w)
+        if not fps:
+            return []
+
+        db = self._mongo._get_db()
+        exclude = {"doc_id": {"$ne": self._doc_id}}
+        if self._job_id:
+            exclude["job_id"] = {"$ne": self._job_id}
+        query = {"fingerprints": {"$in": list(fps)}, **exclude}
+
+        rows = await db.corpus_fingerprints.find(query).limit(limit).to_list(limit)
+        candidates: list[Candidate] = []
+        for row in rows:
+            candidates.append(
+                Candidate(
+                    candidate_id=row["doc_id"],
+                    title="",
+                    url="",
+                    provider=self.provider,
+                    bucket=self.bucket,
+                    submitted_at=_submitted_label(row.get("created_at")),
+                )
+            )
+        return candidates
+
+    async def fetch_text(self, candidate: Candidate) -> str | None:
+        db = self._mongo._get_db()
+        row = await db.corpus_texts.find_one({"doc_id": candidate.candidate_id})
+        return row["normalized_text"] if row else None
+
+    def build_source_match(
+        self,
+        candidate: Candidate,
+        *,
+        source_index: int,
+        matched_words: int,
+        percent: int,
+        spans,
+    ) -> SourceMatch:
+        """Construct a SourceMatch with the privacy boundary applied."""
+        neutral = candidate.submitted_at or _submitted_label(None)
+        cleaned_spans = []
+        for span in spans:
+            cleaned_spans.append(
+                span.model_copy(update={"source_excerpt": ""})
+            )
+        return SourceMatch(
+            source_index=source_index,
+            bucket=self.bucket,
+            title=neutral,
+            url=neutral,
+            doi=None,
+            authors=[],
+            year=None,
+            provider=self.provider,
+            display_label=neutral,
+            matched_words=matched_words,
+            percent=percent,
+            spans=cleaned_spans,
+        )

similarity/corpus/openalex.py

@@ -0,0 +1,85 @@
+"""OpenAlex provider (plan.md §7.3).
+
+Abstract-only matching, reusing the shared `_search_openalex` verbatim. An
+abstract-level match is real but small, so these mostly populate the
+`PUBLICATION` bucket with 1% entries - exactly what the reference screenshot
+shows for Springer. `source_for` + `DOI_PUBLISHER` supply the display label.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+
+from services.academic_search import DOI_PUBLISHER, _search_openalex, source_for
+from similarity.corpus.base import Candidate
+from similarity.schema import Bucket
+from utils.logger import get_logger
+from utils.retry import async_retry
+
+logger = get_logger("openalex_provider")
+
+# Shared throttle across all providers (plan.md §7.1).
+_OPENALEX_SEM = asyncio.Semaphore(2)
+
+
+class OpenAlexProvider:
+    """Search OpenAlex metadata; the abstract is the only retrievable text."""
+
+    name = "openalex"
+    provider = "OpenAlex"
+    bucket = Bucket.PUBLICATION
+
+    def __init__(
+        self,
+        client: httpx.AsyncClient | None = None,
+        max_attempts: int = 3,
+        delay: float = 1.0,
+        backoff: float = 2.0,
+    ):
+        self._client = client or httpx.AsyncClient()
+        self.last_error: str | None = None
+        self._search_once = async_retry(
+            max_attempts=max_attempts, delay=delay, backoff=backoff,
+            exceptions=(httpx.HTTPStatusError, httpx.TransportError),
+        )(self._search_once_raw)
+
+    async def search_phrase(self, phrase: str, limit: int) -> list[Candidate]:
+        try:
+            results = await self._search_once(phrase, limit)
+        except Exception as e:
+            self.last_error = str(e)
+            logger.warning(f"OpenAlex search failed for {phrase[:40]!r}: {e}")
+            return []
+        candidates: list[Candidate] = []
+        for r in results:
+            url = (r.get("url") or "").strip()
+            if not url:
+                continue
+            title = r.get("title", "")
+            doi = r.get("doi") or None
+            label = DOI_PUBLISHER.get((doi or "").split(".")[0] + "." + (doi or "").split(".")[1], "OpenAlex") if doi and doi.startswith("10.") else source_for(r)
+            candidates.append(
+                Candidate(
+                    candidate_id=r.get("paper_id") or url,
+                    title=title,
+                    url=url,
+                    doi=doi,
+                    authors=list(r.get("authors") or []),
+                    year=r.get("year"),
+                    provider=self.provider,
+                    bucket=self.bucket,
+                    text=r.get("abstract") or None,
+                )
+            )
+        return candidates[:limit]
+
+    async def fetch_text(self, candidate: Candidate) -> str | None:
+        """OpenAlex serves abstracts only - there is no full text to fetch."""
+        return candidate.text
+
+    async def _search_once_raw(self, phrase: str, limit: int) -> list[dict]:
+        async with _OPENALEX_SEM:
+            await asyncio.sleep(0.25)
+            return await _search_openalex(self._client, phrase, limit)

similarity/exclusions.py

@@ -0,0 +1,129 @@
+"""Exclusion-region detection for the similarity denominator.
+
+Plan.md §4.1: bibliography, quoted text, title block, equations and captions
+are all excluded by default (each independently toggleable) because they are
+*expected* to coincide with other papers - reference strings are supposed to
+be identical to the original, notation collides across a field, addresses
+match thousands of papers. The remaining words are the ones a similarity
+percentage can honestly claim anything about.
+"""
+
+from __future__ import annotations
+
+import bisect
+import re
+from dataclasses import dataclass
+
+from .normalize import NormalizedDoc
+
+_QUOTE_RE = re.compile(r'"(?:[^"\\]|\\.)*"')
+_BLOCKQUOTE_RE = re.compile(r"^\s*>+\s?.*$", re.MULTILINE)
+_REFERENCE_HEADING_RE = re.compile(r"^\s*(?:\d+\.?\s+)?references?\b", re.IGNORECASE | re.MULTILINE)
+_ABSTRACT_HEADING_RE = re.compile(r"^\s*(?:\d+\.?\s+)?abstract\b", re.IGNORECASE | re.MULTILINE)
+_INTRO_HEADING_RE = re.compile(
+    r"^\s*(?:\d+\.?\s+)?introduction\b", re.IGNORECASE | re.MULTILINE
+)
+# An equation-looking span: an "=" (or LaTeX construct) with a short tail to
+# the end of the line. Equations are excluded because notation collides across
+# a whole field, not because they carry similarity signal.
+_EQUATION_LINE_RE = re.compile(r"[^=\n]{0,60}=[^=\n]{0,160}$", re.MULTILINE)
+_LATEX_LINE_RE = re.compile(r"\\frac|\\sum|\\prod|\\begin\{|\\[a-zA-Z]+\{|\$\$", re.MULTILINE)
+_CAPTION_RE = re.compile(r"^\s*(?:figure|fig\.?|table)\s*\d+[:.]", re.IGNORECASE | re.MULTILINE)
+
+
+@dataclass
+class Exclusions:
+    """Word-index ranges that must not contribute to the denominator."""
+
+    ranges: list[tuple[str, int, int]]   # (name, start_word, end_word)
+
+    def covers(self, word_index: int) -> bool:
+        for _, start, end in self.ranges:
+            if start <= word_index < end:
+                return True
+        return False
+
+    def names(self) -> set[str]:
+        return {name for name, _, _ in self.ranges}
+
+    def word_indices(self) -> set[int]:
+        out: set[int] = set()
+        for _, start, end in self.ranges:
+            out.update(range(start, end))
+        return out
+
+    def count_words(self) -> int:
+        return len(self.word_indices())
+
+
+def _word_range(nd: NormalizedDoc, char_start: int, char_end: int) -> tuple[int, int]:
+    """Inclusive-exclusive word index range covering a char span."""
+    if not nd.words:
+        return (0, 0)
+    start = bisect.bisect_right(nd.starts, char_start) - 1
+    end = bisect.bisect_left(nd.starts, char_end)
+    return (max(0, start), min(len(nd.words), end))
+
+
+def _word_range_from_char(nd: NormalizedDoc, char_pos: int) -> int:
+    """First word index whose char start is >= char_pos."""
+    return bisect.bisect_left(nd.starts, char_pos)
+
+
+def find_exclusions(
+    nd: NormalizedDoc,
+    *,
+    exclude_bibliography: bool = True,
+    exclude_quotes: bool = True,
+    exclude_title_block: bool = True,
+    exclude_equations: bool = True,
+    exclude_captions: bool = True,
+) -> Exclusions:
+    """Detect excluded regions and return them as word-index ranges."""
+    text = nd.original_text
+    ranges: list[tuple[str, int, int]] = []
+
+    if exclude_bibliography:
+        m = _REFERENCE_HEADING_RE.search(text)
+        if m:
+            start_w = _word_range_from_char(nd, m.start())
+            if start_w < len(nd.words):
+                ranges.append(("bibliography", start_w, len(nd.words)))
+
+    if exclude_quotes:
+        for m in _QUOTE_RE.finditer(text):
+            ranges.append(("quotes", *_word_range(nd, m.start(), m.end())))
+        for m in _BLOCKQUOTE_RE.finditer(text):
+            ranges.append(("quotes", *_word_range(nd, m.start(), m.end())))
+
+    if exclude_title_block:
+        heading = _ABSTRACT_HEADING_RE.search(text) or _INTRO_HEADING_RE.search(text)
+        if heading:
+            start_w = _word_range_from_char(nd, heading.start())
+            if 0 < start_w < len(nd.words):
+                ranges.append(("title_block", 0, start_w))
+
+    if exclude_equations:
+        for m in _EQUATION_LINE_RE.finditer(text):
+            ranges.append(("equations", *_word_range(nd, m.start(), m.end())))
+        for m in _LATEX_LINE_RE.finditer(text):
+            ranges.append(("equations", *_word_range(nd, m.start(), m.end())))
+
+    if exclude_captions:
+        for m in _CAPTION_RE.finditer(text):
+            ranges.append(("captions", *_word_range(nd, m.start(), m.end())))
+
+    return Exclusions(ranges=ranges)
+
+
+def _merge(ranges: list[tuple[str, int, int]]) -> list[tuple[str, int, int]]:
+    """Sort and merge overlapping ranges (name from the first/leftmost)."""
+    ordered = sorted((s, e, n) for n, s, e in ranges if e > s)
+    merged: list[tuple[str, int, int]] = []
+    for s, e, n in ordered:
+        if merged and s <= merged[-1][2]:
+            prev_n, prev_s, prev_e = merged[-1]
+            merged[-1] = (prev_n, prev_s, max(prev_e, e))
+        else:
+            merged.append((n, s, e))
+    return merged

similarity/fingerprint.py

@@ -0,0 +1,69 @@
+"""k-gram hashing and winnowing (Schleimer, Wilkerson & Aiken 2003).
+
+The detection guarantee that underpins every recall claim in plan.md:
+
+    any shared contiguous run of >= w + k - 1 words produces an intersecting
+    fingerprint set.
+
+Sketch: a shared run of `w + k - 1` words contains exactly `w` consecutive
+k-grams. In each document there is a window made up entirely of those w shared
+k-grams, and both documents select the minimum of that same multiset of hashes
+- a common value. The property test in test_fingerprint.py pins this down
+against randomized inputs.
+
+The hash is deliberately NOT Python's built-in `hash()`: that is randomized
+per process (PYTHONHASHSEED), so fingerprints stored in the corpus would not
+be reproducible across runs. An FNV-1a over lowercased words keeps values
+stable, deterministic, and case-insensitive.
+"""
+
+from __future__ import annotations
+
+_FNV_OFFSET = 0xCBF29CE484222325
+_FNV_PRIME = 0x100000001B3
+_MASK = 0xFFFFFFFFFFFFFFFF
+
+
+def _stable_hash(words: tuple[str, ...]) -> int:
+    h = _FNV_OFFSET
+    for w in words:
+        for ch in w.lower():
+            h ^= ord(ch)
+            h = (h * _FNV_PRIME) & _MASK
+    return h
+
+
+def kgram_hashes(words: list[str], k: int) -> list[int]:
+    """Hashes of every sliding k-gram in `words`, in document order."""
+    if k <= 0 or len(words) < k:
+        return []
+    return [_stable_hash(tuple(words[i:i + k])) for i in range(len(words) - k + 1)]
+
+
+def winnow(hashes: list[int], w: int) -> set[int]:
+    """Select the minimum of every window of w consecutive k-gram hashes,
+    deduplicating consecutive selections of the same position.
+
+    This is the winnowing step: one fingerprint per window (at most), instead
+    of one per k-gram.
+    """
+    if not hashes or w <= 0:
+        return set()
+    if w == 1:
+        return set(hashes)
+
+    selected: set[int] = set()
+    prev_pos: int | None = None
+    for i in range(len(hashes) - w + 1):
+        window = hashes[i:i + w]
+        min_h = min(window)
+        min_pos = i + window.index(min_h)
+        if min_pos != prev_pos:
+            selected.add(min_h)
+            prev_pos = min_pos
+    return selected
+
+
+def fingerprint(words: list[str], k: int, w: int) -> set[int]:
+    """The full fingerprint of a document: k-grams, then winnowed."""
+    return winnow(kgram_hashes(words, k), w)

similarity/index_writer.py

@@ -0,0 +1,98 @@
+"""Write path into the internal corpus (plan.md §7.4).
+
+Gated on `SIM_INDEX_UPLOADS`, which ships `false`: indexing a user's paper so
+it can be matched against other users' papers is a consent decision (plan.md
+§16 risk 6), not a default. The index itself lives in Mongo collections
+`corpus_fingerprints` and `corpus_texts` (plan.md §6.2); the multikey index on
+`fingerprints` makes the read path a single indexed query.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from similarity.config import settings
+from similarity.fingerprint import fingerprint
+from similarity.normalize import normalize
+
+COLL_FINGERPRINTS = "corpus_fingerprints"
+COLL_TEXTS = "corpus_texts"
+
+
+async def ensure_corpus_indexes(mongo) -> None:
+    """Create the multikey index on `fingerprints` at startup.
+
+    Called once from the orchestrator beside the existing Neo4j index
+    creation - never from the per-query hot path.
+    """
+    db = mongo._get_db()
+    await getattr(db, COLL_FINGERPRINTS).create_index([("fingerprints", 1)])
+
+
+async def index_document(
+    mongo,
+    *,
+    doc_id: str,
+    job_id: str,
+    title: str,
+    owner_email: str,
+    text: str,
+    k: int | None = None,
+    w: int | None = None,
+) -> bool:
+    """Normalize + fingerprint `text` and insert it into the corpus.
+
+    Returns False when indexing is disabled (the default) or the document is
+    too short to fingerprint, True when the rows were written.
+    """
+    if not settings.index_uploads:
+        return False
+
+    k = k or settings.kgram_size
+    w = w or settings.window_size
+    nd = normalize(text)
+    fps = fingerprint(nd.words, k, w)
+    if not fps:
+        return False
+
+    db = mongo._get_db()
+    await getattr(db, COLL_FINGERPRINTS).insert_one(
+        {
+            "doc_id": doc_id,
+            "job_id": job_id,
+            "title": title,
+            "owner_email": owner_email,
+            "word_count": len(nd.words),
+            "fingerprints": list(fps),
+            "created_at": datetime.utcnow(),
+        }
+    )
+    await getattr(db, COLL_TEXTS).insert_one(
+        {
+            "doc_id": doc_id,
+            "normalized_text": " ".join(nd.words),
+            "offset_map": {"starts": nd.starts, "ends": nd.ends},
+        }
+    )
+    return True
+
+
+async def purge_corpus_for_job(mongo, *, job_id: str, doc_id: str | None) -> None:
+    """Remove every corpus row belonging to a deleted job (plan.md §16 risk 6).
+
+    Called from `DELETE /jobs/{job_id}`. Fingerprints are keyed by both
+    job_id and doc_id; texts only by doc_id - delete on both axes.
+    """
+    db = mongo._get_db()
+    fp_filter: dict = {}
+    if job_id and doc_id:
+        fp_filter = {"$or": [{"job_id": job_id}, {"doc_id": doc_id}]}
+    elif job_id:
+        fp_filter = {"job_id": job_id}
+    elif doc_id:
+        fp_filter = {"doc_id": doc_id}
+    else:
+        return
+    await getattr(db, COLL_FINGERPRINTS).delete_many(fp_filter)
+    if doc_id:
+        await getattr(db, COLL_TEXTS).delete_many({"doc_id": doc_id})

similarity/matcher.py

@@ -0,0 +1,136 @@
+"""Verification: fingerprint seeds -> exact, extended match spans.
+
+Stage 2 of the two-stage design. Stage 1 (remote retrieval) only proposes
+candidates; this module proves a match locally with exact word and character
+offsets. A seed found via fingerprint intersection is extended forward and
+backward while consecutive words are identical, and overlapping seeds from the
+same source are merged into one maximal span.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .fingerprint import fingerprint, kgram_hashes
+from .normalize import NormalizedDoc
+
+_EXCERPT_LIMIT = 300
+
+
+@dataclass
+class MatchedSpan:
+    """A verified verbatim overlap between the document and one source."""
+
+    doc_start_word: int      # inclusive
+    doc_end_word: int        # exclusive
+    doc_char_start: int      # offset into the ORIGINAL document text
+    doc_char_end: int
+    word_count: int
+    excerpt: str             # <= 300 chars
+    source_excerpt: str      # matching text in the source
+
+
+def _excerpt(text: str, limit: int = _EXCERPT_LIMIT) -> str:
+    collapsed = " ".join(text.split())
+    return collapsed[:limit] + ("..." if len(collapsed) > limit else "")
+
+
+def _position_index(words: list[str], k: int) -> dict[int, list[int]]:
+    """hash -> list of k-gram start-word indices (only for unique-ish lookup)."""
+    index: dict[int, list[int]] = {}
+    for i, h in enumerate(kgram_hashes(words, k)):
+        index.setdefault(h, []).append(i)
+    return index
+
+
+def match_documents(
+    nd_doc: NormalizedDoc,
+    nd_src: NormalizedDoc,
+    *,
+    k: int,
+    w: int,
+    min_match: int,
+) -> list[MatchedSpan]:
+    """Return all verified verbatim spans of the doc inside the source."""
+    if len(nd_doc.words) < k or len(nd_src.words) < k:
+        return []
+
+    fp_doc = fingerprint(nd_doc.words, k, w)
+    fp_src = fingerprint(nd_src.words, k, w)
+    common = fp_doc & fp_src
+    if not common:
+        return []
+
+    doc_pos = _position_index(nd_doc.words, k)
+    src_pos = _position_index(nd_src.words, k)
+
+    raw: list[tuple[int, int]] = []  # (doc_start, doc_end) candidates
+    for h in common:
+        for dp in doc_pos.get(h, []):
+            for sp in src_pos.get(h, []):
+                start_d, start_s = _extend_backward(nd_doc, nd_src, dp, sp, k)
+                end_d, _ = _extend_forward(nd_doc, nd_src, start_d, start_s, k)
+                raw.append((start_d, end_d))
+
+    if not raw:
+        return []
+
+    merged = _merge_spans(raw)
+    spans: list[MatchedSpan] = []
+    for start_w, end_w in merged:
+        count = end_w - start_w
+        if count < min_match:
+            continue
+        cstart, cend = nd_doc.char_span(start_w, end_w)
+        s_excerpt = _excerpt(" ".join(nd_src.words[start_w:end_w])) if start_w < len(nd_src.words) else ""
+        spans.append(
+            MatchedSpan(
+                doc_start_word=start_w,
+                doc_end_word=end_w,
+                doc_char_start=cstart,
+                doc_char_end=cend,
+                word_count=count,
+                excerpt=_excerpt(nd_doc.original_text[cstart:cend]),
+                source_excerpt=s_excerpt,
+            )
+        )
+    return spans
+
+
+def _extend_backward(
+    nd_doc: NormalizedDoc, nd_src: NormalizedDoc, dp: int, sp: int, k: int
+) -> tuple[int, int]:
+    while dp > 0 and sp > 0 and nd_doc.words[dp - 1].lower() == nd_src.words[sp - 1].lower():
+        dp -= 1
+        sp -= 1
+    return dp, sp
+
+
+def _extend_forward(
+    nd_doc: NormalizedDoc, nd_src: NormalizedDoc, dp: int, sp: int, k: int
+) -> tuple[int, int]:
+    """dp/sp point at a matching k-gram; extend past its k words."""
+    end_d = dp + k
+    end_s = sp + k
+    while (
+        end_d < len(nd_doc.words)
+        and end_s < len(nd_src.words)
+        and nd_doc.words[end_d].lower() == nd_src.words[end_s].lower()
+    ):
+        end_d += 1
+        end_s += 1
+    return end_d, end_s
+
+
+def _merge_spans(raw: list[tuple[int, int]]) -> list[tuple[int, int]]:
+    """Merge overlapping word ranges from overlapping seeds."""
+    if not raw:
+        return []
+    ordered = sorted(raw)
+    merged: list[tuple[int, int]] = []
+    for start, end in ordered:
+        if merged and start <= merged[-1][1]:
+            merged[-1] = (merged[-1][0], max(merged[-1][1], end))
+        else:
+            merged.append((start, end))
+    return merged

similarity/normalize.py

@@ -0,0 +1,51 @@
+"""Text normalization with a character-offset map back into the original.
+
+The offset map is what makes highlighting land on the right pixels: each word
+carries the char start/end it occupied in the ORIGINAL text, so a matched span
+can be turned into exact character offsets without any re-searching.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+# A "word" is a run of letters/digits (unicode-aware), optionally joined by an
+# apostrophe or hyphen so "state-of-the-art" and "don't" stay one token. The
+# regex only ever matches a contiguous slice of the original string, so the
+# offset map round-trips by construction.
+_WORD_RE = re.compile(r"[^\W_]+(?:['\-\u2019][^\W_]+)*")
+
+
+@dataclass
+class NormalizedDoc:
+    """Tokenized document with char offsets into the original text."""
+
+    original_text: str
+    words: list[str]      # tokens, case preserved as in the source
+    starts: list[int]     # char offset of each token's first char
+    ends: list[int]       # char offset one past each token's last char
+
+    @property
+    def word_count(self) -> int:
+        return len(self.words)
+
+    def char_span(self, start_word: int, end_word: int) -> tuple[int, int]:
+        """Char offsets covering words [start_word, end_word) if available."""
+        if not self.words:
+            return (0, 0)
+        start = self.starts[start_word] if start_word < len(self.words) else 0
+        end = self.ends[end_word - 1] if 0 < end_word <= len(self.words) else len(self.original_text)
+        return (start, end)
+
+
+def normalize(text: str) -> NormalizedDoc:
+    text = text or ""
+    words: list[str] = []
+    starts: list[int] = []
+    ends: list[int] = []
+    for m in _WORD_RE.finditer(text):
+        words.append(m.group(0))
+        starts.append(m.start())
+        ends.append(m.end())
+    return NormalizedDoc(original_text=text, words=words, starts=starts, ends=ends)

similarity/pipeline.py

@@ -0,0 +1,303 @@
+"""The similarity pipeline: normalize -> exclude -> select -> retrieve ->
+verify -> match -> aggregate -> report (plan.md §4, §8).
+
+Everything before retrieval is pure (Phase 0 modules). Retrieval is wrapped in
+a hard `asyncio.wait_for` at `SIM_BUDGET_SECONDS`; on timeout we return
+whatever is verified so far with `status="partial"` and
+`coverage.budget_exhausted=True` — partial evidence, honestly labelled, beats
+a hung job.
+
+Provider failures are never swallowed: every provider's `last_error` lands in
+`coverage.providers_failed`. The preprint self-match guard (§16 risk 2) drops
+candidates that are the uploaded paper's own arXiv preprint BEFORE scoring.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+
+from similarity.aggregate import Match, compute_percentages
+from similarity.config import settings
+from similarity.corpus.base import Candidate, CorpusProvider
+from similarity.exclusions import find_exclusions
+from similarity.matcher import match_documents
+from similarity.normalize import normalize
+from similarity.schema import (
+    Bucket,
+    Coverage,
+    MatchSpan,
+    SimilarityReport,
+    SourceMatch,
+)
+from similarity.selector import select_phrases
+
+_WORD_RE = re.compile(r"[^\W_]+")
+
+
+def _token_overlap(a: str, b: str) -> float:
+    ta = set(_WORD_RE.findall((a or "").lower()))
+    tb = set(_WORD_RE.findall((b or "").lower()))
+    if not ta or not tb:
+        return 0.0
+    return len(ta & tb) / min(len(ta), len(tb))
+
+
+def _author_overlap(a: list[str], b: list[str]) -> float:
+    sa = {x.lower() for x in (a or [])}
+    sb = {x.lower() for x in (b or [])}
+    if not sa or not sb:
+        return 0.0
+    return len(sa & sb) / max(len(sa), len(sb))
+
+
+def is_preprint_self_match(
+    *,
+    doc_title: str,
+    cand_title: str,
+    cand_doi: str | None,
+    cand_authors: list[str],
+    similarity: int,
+    doc_authors: list[str] | None = None,
+    doc_doi: str | None = None,
+) -> bool:
+    """plan.md §16 risk 2: drop the paper's own preprint before scoring."""
+    if doc_doi and cand_doi and doc_doi.lower() == cand_doi.lower():
+        return True
+    if _token_overlap(doc_title, cand_title) >= 0.9:
+        return True
+    if similarity > 60 and _author_overlap(doc_authors or [], cand_authors) >= 0.5:
+        return True
+    return False
+
+
+def _excerpt(text: str, limit: int = 300) -> str:
+    collapsed = " ".join(text.split())
+    return collapsed[:limit] + ("..." if len(collapsed) > limit else "")
+
+
+class SimilarityPipeline:
+    """Coordinates the corpus providers and the pure scoring core."""
+
+    def __init__(
+        self,
+        providers: list | None = None,
+        *,
+        k: int | None = None,
+        w: int | None = None,
+        min_match: int | None = None,
+        max_queries: int | None = None,
+        budget_seconds: float | None = None,
+        doc_title: str = "",
+        doc_doi: str | None = None,
+        doc_authors: list[str] | None = None,
+        embedder=None,
+    ):
+        self.providers = providers or []
+        self.k = k or settings.kgram_size
+        self.w = w or settings.window_size
+        self.min_match = min_match or settings.min_match_words
+        self.max_queries = max_queries or settings.max_queries
+        self.budget_seconds = budget_seconds if budget_seconds is not None else settings.budget_seconds
+        self.doc_title = doc_title or ""
+        self.doc_doi = doc_doi
+        self.doc_authors = doc_authors or []
+        self.embedder = embedder
+
+    async def run(self, text: str, *, doc_id: str, keywords: list[str] | None = None) -> SimilarityReport:
+        nd = normalize(text)
+        exclusions = find_exclusions(nd)
+        total = nd.word_count - exclusions.count_words()
+
+        if total < settings.min_document_words:
+            return SimilarityReport(
+                overall_percent=0,
+                coverage=Coverage(
+                    total_words_compared=total,
+                    words_excluded=exclusions.count_words(),
+                    exclusions_applied=sorted(exclusions.names()),
+                    providers_queried=[p.name for p in self.providers],
+                    phrases_queried=0,
+                    candidates_retrieved=0,
+                    candidates_verified=0,
+                    internal_corpus_size=0,
+                    min_match_words=self.min_match,
+                ),
+                status="unavailable",
+                notes=[
+                    f"Document too short ({total} comparable words < "
+                    f"{settings.min_document_words}) - no number is rendered."
+                ],
+            )
+
+        phrases = select_phrases(nd, exclusions, max_queries=self.max_queries)
+        coverage = Coverage(
+            total_words_compared=total,
+            words_excluded=exclusions.count_words(),
+            exclusions_applied=sorted(exclusions.names()),
+            providers_queried=[p.name for p in self.providers],
+            phrases_queried=len(phrases),
+            candidates_retrieved=0,
+            candidates_verified=0,
+            internal_corpus_size=0,
+            min_match_words=self.min_match,
+        )
+
+        try:
+            matches, sources, notes, retrieved, verified, spans_by_source, source_providers = await asyncio.wait_for(
+                self._retrieve_and_verify(nd, phrases, coverage),
+                timeout=self.budget_seconds,
+            )
+            budget_exhausted = False
+        except asyncio.TimeoutError:
+            matches, sources, notes, retrieved, verified, spans_by_source, source_providers = [], {}, [], 0, 0, {}, {}
+            budget_exhausted = True
+
+        coverage.candidates_retrieved = retrieved
+        coverage.candidates_verified = verified
+        coverage.budget_exhausted = budget_exhausted
+
+        result = compute_percentages(total, matches, min_match_words=self.min_match)
+
+        source_matches = self._assemble_sources(sources, spans_by_source, source_providers, result.per_source, total)
+
+        status = "complete"
+        if coverage.providers_failed or budget_exhausted:
+            status = "partial"
+
+        report = SimilarityReport(
+            overall_percent=result.overall_percent,
+            bucket_percents=result.bucket_percents,
+            sources=source_matches,
+            coverage=coverage,
+            status=status,
+            notes=notes,
+        )
+
+        # Paraphrase pass is decoration only - it must never move the headline.
+        from similarity.semantic import semantic_pass
+        report = semantic_pass(report, doc_text=text, embedder=self.embedder)
+
+        return report
+
+    async def _retrieve_and_verify(self, nd, phrases, coverage):
+        matches: list[Match] = []
+        spans_by_source: dict[int, list[MatchSpan]] = {}
+        source_candidates: dict[int, Candidate] = {}
+        source_providers: dict[int, CorpusProvider] = {}
+        notes: list[str] = []
+        retrieved = 0
+        verified = 0
+        dropped_self = 0
+        next_source_index = 1
+        trust = {Bucket.INTERNAL: 3, Bucket.OPEN_ACCESS: 2, Bucket.PUBLICATION: 1}
+
+        for provider in self.providers:
+            if provider.last_error:
+                coverage.providers_failed.append(provider.name)
+
+        for phrase in phrases:
+            for provider in self.providers:
+                try:
+                    candidates = await provider.search_phrase(phrase, limit=10)
+                except Exception as e:
+                    provider.last_error = str(e)
+                    candidates = []
+                retrieved += len(candidates)
+                if provider.last_error and provider.name not in coverage.providers_failed:
+                    coverage.providers_failed.append(provider.name)
+                for cand in candidates:
+                    if not cand.url and cand.bucket is not Bucket.INTERNAL:
+                        continue  # §6.1: a source you cannot open is not evidence
+                    if is_preprint_self_match(
+                        doc_title=self.doc_title,
+                        cand_title=cand.title,
+                        cand_doi=cand.doi,
+                        cand_authors=cand.authors,
+                        similarity=100,  # conservative: pre-filter before measuring
+                        doc_authors=self.doc_authors,
+                        doc_doi=self.doc_doi,
+                    ):
+                        dropped_self += 1
+                        continue
+                    try:
+                        text = cand.text or await provider.fetch_text(cand)
+                    except Exception as e:
+                        provider.last_error = str(e)
+                        text = None
+                    if not text:
+                        continue
+                    verified += 1
+                    nd_src = normalize(text)
+                    spans = match_documents(nd, nd_src, k=self.k, w=self.w, min_match=self.min_match)
+                    if not spans:
+                        continue
+                    index = next_source_index
+                    next_source_index += 1
+                    source_candidates[index] = cand
+                    source_providers[index] = provider
+                    bucket = cand.bucket or provider.bucket
+                    spans_by_source[index] = [
+                        MatchSpan(
+                            doc_start_word=s.doc_start_word,
+                            doc_end_word=s.doc_end_word,
+                            doc_char_start=s.doc_char_start,
+                            doc_char_end=s.doc_char_end,
+                            word_count=s.word_count,
+                            excerpt=s.excerpt,
+                            source_excerpt=s.source_excerpt,
+                            source_index=index,
+                        )
+                        for s in spans
+                    ]
+                    matches.append(
+                        Match(
+                            source_index=index,
+                            bucket=bucket,
+                            trust=trust.get(bucket, 1),
+                            doc_start=spans[0].doc_start_word,
+                            doc_end=max(s.doc_end_word for s in spans),
+                        )
+                    )
+
+        if dropped_self:
+            notes.append(f"{dropped_self} probable preprint/self-citation excluded")
+        return matches, source_candidates, notes, retrieved, verified, spans_by_source, source_providers
+
+    def _assemble_sources(self, source_candidates, spans_by_source, source_providers, per_source, total):
+        sources: list[SourceMatch] = []
+        for index, cand in sorted(source_candidates.items()):
+            words = per_source.get(index, 0)
+            provider = source_providers.get(index)
+            build = getattr(provider, "build_source_match", None)
+            if build is not None:
+                # Internal corpus: the privacy boundary is enforced HERE at
+                # construction (plan.md §7.4) - neutral label, no URL, no
+                # authors, source_excerpt stripped from the spans.
+                sources.append(
+                    build(
+                        cand,
+                        source_index=index,
+                        matched_words=words,
+                        percent=round(100 * words / total) if total else 0,
+                        spans=spans_by_source.get(index, []),
+                    )
+                )
+                continue
+            sources.append(
+                SourceMatch(
+                    source_index=index,
+                    bucket=cand.bucket or Bucket.OPEN_ACCESS,
+                    title=cand.title,
+                    url=cand.url,
+                    doi=cand.doi,
+                    authors=cand.authors,
+                    year=cand.year,
+                    provider=cand.provider or (provider.name if provider else ""),
+                    display_label=cand.url,
+                    matched_words=words,
+                    percent=round(100 * words / total) if total else 0,
+                    spans=spans_by_source.get(index, []),
+                )
+            )
+        return sources

similarity/schema.py

@@ -0,0 +1,82 @@
+"""Pydantic schemas for the corpus-similarity report.
+
+The field names here are the contract: the pipeline emits them, the API serves
+them, and the frontend renders them. Later phases depend on them, so changes
+ripple - keep them in lock-step with plan.md §6.1.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class Bucket(str, Enum):
+    OPEN_ACCESS = "open_access"    # CORE, arXiv - the "Internet Sources" analogue
+    PUBLICATION = "publication"    # Crossref/DOI-bearing, publisher-hosted
+    INTERNAL = "internal"          # prior CitationEdge uploads
+
+
+class MatchSpan(BaseModel):
+    doc_start_word: int             # inclusive, index into post-exclusion word list
+    doc_end_word: int               # exclusive
+    doc_char_start: int             # offset into ORIGINAL text - drives highlighting
+    doc_char_end: int
+    word_count: int
+    excerpt: str = ""               # <= 300 chars, for display
+    source_excerpt: str = ""        # the matching text in the source
+    source_index: int               # 1-based, matches the numbered UI chips
+
+
+class SourceMatch(BaseModel):
+    source_index: int
+    bucket: Bucket
+    title: str
+    url: str                        # ALWAYS resolvable - a source you cannot open
+                                    # is not evidence
+    doi: str | None = None
+    authors: list[str] = Field(default_factory=list)
+    year: int | None = None
+    provider: str                   # "CORE" | "arXiv" | "Crossref" | "OpenAlex" | "CitationEdge Corpus"
+    display_label: str
+    matched_words: int              # attributed words only (plan.md §4.3)
+    percent: int                    # matched_words / total_words * 100, rounded
+    spans: list[MatchSpan] = Field(default_factory=list)
+
+
+class ParaphraseFlag(BaseModel):
+    doc_excerpt: str
+    source_excerpt: str
+    source_index: int
+    cosine: float
+
+
+class Coverage(BaseModel):
+    """Mandatory disclosure - the denominator behind the number."""
+
+    total_words_compared: int
+    words_excluded: int
+    exclusions_applied: list[str] = Field(default_factory=list)
+    providers_queried: list[str] = Field(default_factory=list)
+    providers_failed: list[str] = Field(default_factory=list)  # a 429 must be visible
+    phrases_queried: int
+    candidates_retrieved: int
+    candidates_verified: int
+    internal_corpus_size: int
+    min_match_words: int
+    budget_exhausted: bool = False
+    checked_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class SimilarityReport(BaseModel):
+    overall_percent: int
+    bucket_percents: dict[Bucket, int] = Field(default_factory=dict)
+    sources: list[SourceMatch] = Field(default_factory=list)
+    paraphrase_flags: list[ParaphraseFlag] = Field(default_factory=list)
+    coverage: Coverage
+    status: Literal["complete", "partial", "unavailable"]
+    notes: list[str] = Field(default_factory=list)
+    processing_time_ms: float = 0.0

similarity/selector.py

@@ -0,0 +1,109 @@
+"""Selection of the distinctive phrase queries sent to CORE in stage 1.
+
+Plan.md §3: we cannot query every phrase in a paper, so we pick the most
+*distinctive* ones - rare-in-this-document words are usually rare globally
+too - while spreading the selection across sections so we don't query thirty
+phrases from the introduction and none from the methods. Boilerplate, quoted
+text, and all-stopword phrases never get selected.
+
+The STOPWORDS / FILLER_TOKENS lists are imported from paper_recommender.py,
+never forked - a phrase-selection rule and the recommender must agree on what
+"generic" means.
+"""
+
+from __future__ import annotations
+
+import math
+
+from services.paper_recommender import FILLER_TOKENS, STOPWORDS
+
+from .exclusions import Exclusions
+from .normalize import NormalizedDoc
+
+# Normalized (lowercased) boilerplate prefixes that carry no similarity signal.
+BOILERPLATE_PREFIXES: tuple[str, ...] = (
+    "in this paper we propose",
+    "in this paper, we propose",
+    "in this paper we present",
+    "in this study we",
+    "we present a novel",
+    "the rest of this paper is organized as follows",
+    "this paper is organized as follows",
+    "the remainder of this paper is organized as follows",
+    "in recent years, ",
+    "section x of this paper",
+    "the rest of the paper is organized as follows",
+    "we then discuss experimental findings",
+    "the contributions of this paper are",
+)
+
+# Default phrase length sent to `fullText:"..."` - inside the 8-12 word band.
+DEFAULT_PHRASE_WORDS = 10
+_DEFAULT_BUCKETS = 6
+
+_NOISE = STOPWORDS | FILLER_TOKENS
+
+
+def _is_boilerplate(phrase: str) -> bool:
+    lowered = phrase.lower()
+    for prefix in BOILERPLATE_PREFIXES:
+        if lowered.startswith(prefix):
+            return True
+    return False
+
+
+def select_phrases(
+    nd: NormalizedDoc,
+    exclusions: Exclusions,
+    max_queries: int = 24,
+    phrase_words: int = DEFAULT_PHRASE_WORDS,
+    num_buckets: int = _DEFAULT_BUCKETS,
+) -> list[str]:
+    """Return up to `max_queries` distinctive, stratified query phrases."""
+    words = nd.words
+    if len(words) < phrase_words:
+        return []
+
+    freq: dict[str, int] = {}
+    for w in words:
+        low = w.lower()
+        if low in _NOISE:
+            continue
+        freq[low] = freq.get(low, 0) + 1
+
+    candidates: list[tuple[float, int, str]] = []
+    last_start = len(words) - phrase_words
+    for i in range(last_start + 1):
+        if exclusions.covers(i) or exclusions.covers(i + phrase_words - 1):
+            continue
+        window = words[i:i + phrase_words]
+        phrase = " ".join(window)
+        if _is_boilerplate(phrase):
+            continue
+        lowered = [w.lower() for w in window]
+        content = [t for t in lowered if t not in _NOISE]
+        if not content or len(content) < phrase_words // 2:
+            continue
+        # mean inverse token frequency within the document
+        score = sum(1.0 / (1.0 + freq.get(t, 0)) for t in content) / len(content)
+        candidates.append((score, i, phrase))
+
+    if not candidates:
+        return []
+
+    # Stratify: chunk candidates by start position, keep the best of each.
+    total = len(candidates)
+    buckets: list[list[tuple[float, int, str]]] = [[] for _ in range(num_buckets)]
+    for cand in candidates:
+        _, pos, _ = cand
+        idx = min(num_buckets - 1, pos * num_buckets // max(1, total))
+        buckets[idx].append(cand)
+
+    per_bucket = math.ceil(max_queries / num_buckets)
+    picked: list[tuple[float, int, str]] = []
+    for bucket in buckets:
+        bucket.sort(reverse=True)  # by (score, pos, phrase)
+        picked.extend(bucket[:per_bucket])
+
+    picked.sort(reverse=True)
+    return [phrase for _, _, phrase in picked[:max_queries]]

similarity/semantic.py

@@ -0,0 +1,54 @@
+"""Paraphrase pass (plan.md §4, prompt Phase 3).
+
+Verbatin matching decides the headline number. The paraphrase pass is a
+human-review aid only: it embeds each matched span's document excerpt against
+the source excerpt and, when the cosine similarity is suspiciously high, emits
+a `ParaphraseFlag` for the frontend to show as "similar wording / possible
+paraphrase - verify". It NEVER touches `overall_percent` or
+`bucket_percents`, so a fuzzy embedding can never inflate the corpus claim.
+"""
+
+from __future__ import annotations
+
+from similarity.config import settings
+from similarity.schema import ParaphraseFlag, SimilarityReport
+
+
+def semantic_pass(
+    report: SimilarityReport,
+    *,
+    doc_text: str | None = None,
+    embedder=None,
+) -> SimilarityReport:
+    """Return a copy of `report` with `paraphrase_flags` populated.
+
+    The pass only runs when enabled AND an embedder is supplied - the embedder
+    is heavy (SciBERT) and is constructed by the caller (the agent wiring),
+    never implicitly. Without one the report is returned unchanged.
+    """
+    if not settings.enable_paraphrase or embedder is None:
+        return report
+
+    flags: list[ParaphraseFlag] = []
+    for source in report.sources:
+        for span in source.spans:
+            doc_excerpt = span.excerpt or (doc_text or "")[:300]
+            if not doc_excerpt or not span.source_excerpt:
+                continue
+            try:
+                dv = embedder.embed(doc_excerpt)
+                sv = embedder.embed(span.source_excerpt)
+                cosine = embedder.cosine_similarity(dv, sv)
+            except Exception:
+                continue  # one bad span must not kill the whole report
+            if cosine >= settings.paraphrase_threshold:
+                flags.append(
+                    ParaphraseFlag(
+                        doc_excerpt=doc_excerpt,
+                        source_excerpt=span.source_excerpt,
+                        source_index=span.source_index,
+                        cosine=round(cosine, 4),
+                    )
+                )
+
+    return report.model_copy(update={"paraphrase_flags": flags})

tests/similarity/conftest.py

@@ -0,0 +1,115 @@
+"""Shared fixtures for similarity Phase 1 tests.
+
+A tiny in-memory Mongo stand-in so corpus write/read/purge paths run without
+a real database, mirroring how backend/tests/conftest.py keeps heavy
+dependencies out of the tests.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from similarity.corpus.internal import InternalCorpusProvider
+from similarity.index_writer import COLL_FINGERPRINTS, COLL_TEXTS
+
+
+class _FakeCursor:
+    def __init__(self, docs, limit):
+        self._docs = docs
+        self._limit = limit
+
+    def limit(self, n):
+        return _FakeCursor(self._docs, n)
+
+    async def to_list(self, length=None):
+        return list(self._docs[: self._limit])
+
+
+class _FakeCollection:
+    def __init__(self):
+        self._docs: list[dict] = []
+        self._indexes: list[list] = []
+
+    async def create_index(self, keys, **kwargs):
+        self._indexes.append(keys)
+
+    async def insert_one(self, doc):
+        self._docs.append(dict(doc))
+        return {"inserted_id": "fake"}
+
+    async def delete_many(self, query):
+        kept = []
+        for d in self._docs:
+            if self._matches(d, query):
+                continue
+            kept.append(d)
+        removed = len(self._docs) - len(kept)
+        self._docs = kept
+        return {"deleted_count": removed}
+
+    async def find_one(self, query):
+        for d in self._docs:
+            if self._matches(d, query):
+                return dict(d)
+        return None
+
+    def find(self, query, projection=None):
+        matched = [dict(d) for d in self._docs if self._matches(d, query)]
+        if projection and projection.get("_id") == 0:
+            for d in matched:
+                d.pop("_id", None)
+        return _FakeCursor(matched, limit=len(matched))
+
+    @staticmethod
+    def _matches(doc, query):
+        for key, cond in query.items():
+            if key == "$or":
+                if not any(_FakeCollection._matches(doc, sub) for sub in cond):
+                    return False
+                continue
+            value = doc.get(key)
+            if isinstance(cond, dict):
+                if "$in" in cond:
+                    if not value or not any(h in value for h in cond["$in"]):
+                        return False
+                elif "$ne" in cond and value == cond["$ne"]:
+                    return False
+            elif value != cond:
+                return False
+        return True
+
+
+class FakeMongo:
+    """MongoService-shaped object; tests the collections the corpus uses."""
+
+    def __init__(self):
+        self._collections = {COLL_FINGERPRINTS: _FakeCollection(), COLL_TEXTS: _FakeCollection()}
+
+    def _get_db(self):
+        return self
+
+    def __getitem__(self, name):
+        return self._collections[name]
+
+    # db.collection attribute access
+    @property
+    def corpus_fingerprints(self):
+        return self._collections[COLL_FINGERPRINTS]
+
+    @property
+    def corpus_texts(self):
+        return self._collections[COLL_TEXTS]
+
+
+@pytest.fixture
+def fake_mongo():
+    return FakeMongo()
+
+
+@pytest.fixture
+def internal_provider(fake_mongo):
+    def _make(*, doc_id="doc_new", job_id=None, k=5, w=4):
+        return InternalCorpusProvider(
+            fake_mongo, doc_id=doc_id, job_id=job_id, k=k, w=w
+        )
+    return _make

tests/similarity/test_aggregate.py

@@ -0,0 +1,109 @@
+"""Aggregation tests - the scoring spec from plan.md §4, case by case.
+
+The single invariant that must never break: a word is counted once no matter
+how many sources contain it. Every other rule (attribution, buckets, minimum
+length) is layered on top of that bitmap.
+"""
+from similarity.aggregate import Match, compute_percentages
+from similarity.schema import Bucket
+
+INTERNAL = Bucket.INTERNAL
+OPEN = Bucket.OPEN_ACCESS
+PUB = Bucket.PUBLICATION
+
+
+def test_identical_document_scores_100():
+    # a single match covering every word -> 100%
+    total = 10
+    matches = [Match(source_index=1, bucket=OPEN, trust=1, doc_start=0, doc_end=10)]
+    result = compute_percentages(total, matches)
+    assert result.overall_percent == 100
+
+
+def test_disjoint_text_scores_0():
+    total = 10
+    result = compute_percentages(total, [])
+    assert result.overall_percent == 0
+    assert result.per_source == {}
+    assert result.bucket_words == {}
+    assert result.bucket_percents == {}
+
+
+def test_one_span_claimed_by_three_sources_counts_once():
+    """The double-count regression: one span in three sources must not inflate
+    the number - a word is covered or it is not."""
+    total = 10
+    matches = [
+        Match(source_index=1, bucket=OPEN, trust=1, doc_start=0, doc_end=10),
+        Match(source_index=2, bucket=PUB, trust=1, doc_start=0, doc_end=10),
+        Match(source_index=3, bucket=INTERNAL, trust=1, doc_start=0, doc_end=10),
+    ]
+    result = compute_percentages(total, matches)
+    assert result.overall_percent == 100
+    assert sum(result.per_source.values()) == total
+
+
+def test_attribution_goes_to_longest_match():
+    total = 10
+    matches = [
+        Match(source_index=1, bucket=OPEN, trust=1, doc_start=0, doc_end=10),
+        Match(source_index=2, bucket=PUB, trust=1, doc_start=2, doc_end=5),
+    ]
+    result = compute_percentages(total, matches)
+    # the shorter overlapping match owns nothing; its words are attributed to
+    # the source with the longer confirmed match
+    assert result.per_source == {1: 10}
+
+
+def test_ties_break_on_higher_trust():
+    total = 10
+    matches = [
+        Match(source_index=1, bucket=OPEN, trust=1, doc_start=0, doc_end=10),
+        Match(source_index=2, bucket=INTERNAL, trust=2, doc_start=0, doc_end=10),
+    ]
+    result = compute_percentages(total, matches)
+    assert result.per_source == {2: 10}
+
+
+def test_bucket_percentages_are_independent_and_may_exceed_headline():
+    total = 100
+    matches = [
+        # both buckets cover the same 30 words -> each bucket 30%, headline 30%
+        Match(source_index=1, bucket=OPEN, trust=1, doc_start=0, doc_end=30),
+        Match(source_index=2, bucket=PUB, trust=1, doc_start=0, doc_end=30),
+    ]
+    result = compute_percentages(total, matches)
+    assert result.overall_percent == 30
+    assert result.bucket_percents[OPEN] == 30
+    assert result.bucket_percents[PUB] == 30
+
+
+def test_bucket_percents_can_sum_above_headline():
+    total = 100
+    matches = [
+        Match(source_index=1, bucket=OPEN, trust=1, doc_start=0, doc_end=30),
+        Match(source_index=2, bucket=PUB, trust=1, doc_start=30, doc_end=60),
+    ]
+    result = compute_percentages(total, matches)
+    assert result.overall_percent == 60
+    assert result.bucket_percents[OPEN] == 30
+    assert result.bucket_percents[PUB] == 30
+    # deliberately NOT 60: buckets are "as if only this bucket existed"
+    assert result.bucket_percents[OPEN] + result.bucket_percents[PUB] == 60
+
+
+def test_matches_below_minimum_length_are_excluded():
+    """The min-length floor lives at the matcher boundary, but aggregation must
+    be robust to a caller passing short spans anyway - it computes, never
+    assumes the input is pre-filtered."""
+    total = 100
+    matches = [
+        Match(source_index=1, bucket=OPEN, trust=1, doc_start=0, doc_end=2),
+    ]
+    result = compute_percentages(total, matches, min_match_words=8)
+    assert result.overall_percent == 0
+
+
+def test_zero_total_words_returns_zero():
+    result = compute_percentages(0, [])
+    assert result.overall_percent == 0

tests/similarity/test_corpus_providers.py

@@ -0,0 +1,329 @@
+"""Provider tests for CORE, arXiv, OpenAlex, Crossref.
+
+No network: every request goes through `httpx.MockTransport`. Fixtures cover
+the happy path, 429, 5xx, malformed JSON, empty results, and a candidate
+with no resolvable URL (which must be dropped, plan.md §6.1).
+"""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+import pytest
+
+from similarity.corpus.base import Candidate
+from similarity.corpus.core_api import CoreApiProvider
+from similarity.corpus.openalex import OpenAlexProvider
+from similarity.corpus.crossref import CrossrefProvider
+from similarity.corpus.arxiv import ArxivProvider
+from similarity.schema import Bucket
+
+
+@pytest.fixture(autouse=True)
+def _no_sleep(monkeypatch):
+    """The shared @async_retry and per-call stagger sleep on failure paths;
+    collapse them so failure tests run fast without changing behaviour."""
+    async def _instant(delay):
+        return None
+    monkeypatch.setattr("asyncio.sleep", _instant)
+
+
+def _client(handler) -> httpx.AsyncClient:
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+
+# ── CORE ──────────────────────────────────────────────────────────────────────
+
+CORE_WORK = {
+    "id": "C1",
+    "title": "Quantum rings at equilibrium",
+    "downloadUrl": "https://core.ac.uk/download/1.pdf",
+    "doi": "10.1007/x",
+    "yearPublished": 2021,
+    "authors": ["A. Physicist", "B. Theorist"],
+    "fullText": "quantum coherent transport through a one dimensional chain",
+}
+
+
+@pytest.fixture
+def core_client():
+    def handler(request):
+        return httpx.Response(200, json={"results": [CORE_WORK]})
+    return _client(handler)
+
+
+@pytest.fixture
+def core_429():
+    return _client(lambda req: httpx.Response(429))
+
+
+@pytest.fixture
+def core_500():
+    return _client(lambda req: httpx.Response(500))
+
+
+@pytest.fixture
+def core_malformed():
+    return _client(lambda req: httpx.Response(200, text="not json {{{"))
+
+
+@pytest.fixture
+def core_empty():
+    return _client(lambda req: httpx.Response(200, json={"results": []}))
+
+
+async def test_core_happy_path(core_client):
+    p = CoreApiProvider(api_key="k", client=core_client, max_attempts=1)
+    cands = await p.search_phrase("quantum coherent transport", limit=5)
+    assert len(cands) == 1
+    c = cands[0]
+    assert c.candidate_id == "C1"
+    assert c.title == "Quantum rings at equilibrium"
+    assert c.url == "https://core.ac.uk/download/1.pdf"
+    assert c.bucket is Bucket.OPEN_ACCESS
+    assert c.provider == "CORE"
+    # full text inline => fetch is a single round trip
+    assert (await p.fetch_text(c)) == "quantum coherent transport through a one dimensional chain"
+
+
+async def test_core_sends_bearer_when_key_present():
+    captured = {}
+
+    def handler(request):
+        captured["auth"] = request.headers.get("Authorization")
+        return httpx.Response(200, json={"results": []})
+
+    client = _client(handler)
+    p = CoreApiProvider(api_key="secret", client=client, max_attempts=1)
+    await p.search_phrase("some phrase", limit=5)
+    assert captured["auth"] == "Bearer secret"
+
+
+async def test_core_no_key_sends_no_auth():
+    captured = {}
+
+    def handler(request):
+        captured["auth"] = request.headers.get("Authorization")
+        return httpx.Response(200, json={"results": []})
+
+    client = _client(handler)
+    p = CoreApiProvider(api_key="", client=client, max_attempts=1)
+    await p.search_phrase("some phrase", limit=5)
+    assert captured["auth"] is None
+
+
+async def test_core_429_records_error_and_returns_empty(core_429):
+    p = CoreApiProvider(api_key="k", client=core_429, max_attempts=1)
+    assert await p.search_phrase("phrase", limit=5) == []
+    assert p.last_error is not None
+
+
+async def test_core_500_records_error_and_returns_empty(core_500):
+    p = CoreApiProvider(api_key="k", client=core_500, max_attempts=1)
+    assert await p.search_phrase("phrase", limit=5) == []
+    assert p.last_error is not None
+
+
+async def test_core_malformed_json_records_error(core_malformed):
+    p = CoreApiProvider(api_key="k", client=core_malformed, max_attempts=1)
+    assert await p.search_phrase("phrase", limit=5) == []
+    assert p.last_error is not None
+
+
+async def test_core_empty_results(core_empty):
+    p = CoreApiProvider(api_key="k", client=core_empty, max_attempts=1)
+    assert await p.search_phrase("nothing here", limit=5) == []
+
+
+async def test_core_metadata_only_record_falls_back_to_doi_url():
+    def handler(request):
+        return httpx.Response(200, json={"results": [{
+            "id": "C2", "title": "No download link", "doi": "10.1109/xyz",
+            "fullText": "some matched text here",
+        }]})
+    p = CoreApiProvider(api_key="k", client=_client(handler), max_attempts=1)
+    cands = await p.search_phrase("matched text", limit=5)
+    assert cands[0].url == "https://doi.org/10.1109/xyz"
+
+
+async def test_core_candidate_without_url_or_doi_is_dropped():
+    def handler(request):
+        return httpx.Response(200, json={"results": [{
+            "id": "C3", "title": "No url no doi", "fullText": "text",
+        }]})
+    p = CoreApiProvider(api_key="k", client=_client(handler), max_attempts=1)
+    cands = await p.search_phrase("text", limit=5)
+    # the candidate keeps an empty url; the pipeline drops it (test here that
+    # it is representable and url-less, matching §6.1 "candidates without a
+    # URL are dropped" at the pipeline layer).
+    assert len(cands) == 1
+    assert cands[0].url == ""
+
+
+# ── OpenAlex / Crossref ───────────────────────────────────────────────────────
+
+OPENALEX_RESULT = {
+    "display_name": "Attention is all you need",
+    "authorships": [{"author": {"display_name": "A. Vaswani"}}],
+    "publication_year": 2017,
+    "abstract_inverted_index": {"attention": [0], "is": [1], "all": [2]},
+    "cited_by_count": 90000,
+    "doi": "https://doi.org/10.1007/x",
+    "id": "https://openalex.org/W1",
+}
+
+
+@pytest.fixture
+def openalex_client():
+    return _client(lambda req: httpx.Response(200, json={"results": [OPENALEX_RESULT]}))
+
+
+@pytest.fixture
+def crossref_client():
+    return _client(lambda req: httpx.Response(200, json={
+        "message": {"items": [{
+            "type": "journal-article",
+            "title": ["A transformer for text"],
+            "DOI": "10.1016/j.cam.2020.1",
+            "issued": {"date-parts": [[2020]]},
+            "URL": "https://doi.org/10.1016/j.cam.2020.1",
+            "abstract": "<jats:p>transformers are great for sequences</jats:p>",
+            "author": [{"given": "Jane", "family": "Doe"}],
+        }]},
+    }))
+
+
+async def test_openalex_happy_path(openalex_client):
+    p = OpenAlexProvider(client=openalex_client, max_attempts=1)
+    cands = await p.search_phrase("attention is all you need", limit=5)
+    assert len(cands) == 1
+    c = cands[0]
+    assert c.bucket is Bucket.PUBLICATION
+    assert c.provider == "OpenAlex"
+    assert "attention is all" == c.text
+    assert c.doi == "10.1007/x"
+    assert (await p.fetch_text(c)) == c.text
+
+
+async def test_openalex_429(openalex_client):
+    p = OpenAlexProvider(client=_client(lambda req: httpx.Response(429)), max_attempts=1)
+    assert await p.search_phrase("x", limit=5) == []
+    assert p.last_error is not None
+
+
+async def test_openalex_malformed(openalex_client):
+    p = OpenAlexProvider(client=_client(lambda req: httpx.Response(200, text="oops")), max_attempts=1)
+    assert await p.search_phrase("x", limit=5) == []
+    assert p.last_error is not None
+
+
+async def test_crossref_happy_path(crossref_client):
+    p = CrossrefProvider(client=crossref_client, max_attempts=1)
+    cands = await p.search_phrase("transformers for text", limit=5)
+    assert len(cands) == 1
+    c = cands[0]
+    assert c.bucket is Bucket.PUBLICATION
+    assert c.provider == "Crossref"
+    assert c.doi == "10.1016/j.cam.2020.1"
+    assert "transformers" in c.text
+    assert (await p.fetch_text(c)) == c.text
+
+
+async def test_crossref_500(crossref_client):
+    p = CrossrefProvider(client=_client(lambda req: httpx.Response(500)), max_attempts=1)
+    assert await p.search_phrase("x", limit=5) == []
+    assert p.last_error is not None
+
+
+# ── arXiv ─────────────────────────────────────────────────────────────────────
+
+ARXIV_XML = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <entry>
+    <id>http://arxiv.org/abs/2101.00001</id>
+    <published>2021-01-01T00:00:00Z</published>
+    <title>A      test of the arXiv provider end to end</title>
+    <author><name>Alice Adams</name></author>
+    <summary>This is the abstract text that we search on for now.</summary>
+  </entry>
+</feed>
+"""
+
+
+@pytest.fixture
+def arxiv_client():
+    def handler(request):
+        return httpx.Response(200, text=ARXIV_XML)
+    return _client(handler)
+
+
+async def test_arxiv_happy_path(arxiv_client):
+    p = ArxivProvider(client=arxiv_client, max_attempts=1)
+    cands = await p.search_phrase("test of the arxiv provider", limit=5)
+    assert len(cands) == 1
+    c = cands[0]
+    assert c.bucket is Bucket.OPEN_ACCESS
+    assert c.provider == "arXiv"
+    assert "arxiv.org" in c.url
+    assert c.year == 2021
+    assert c.text == "This is the abstract text that we search on for now."
+    assert c.authors == ["Alice Adams"]
+
+
+async def test_arxiv_429(arxiv_client):
+    p = ArxivProvider(client=_client(lambda req: httpx.Response(429)), max_attempts=1)
+    assert await p.search_phrase("x", limit=5) == []
+    assert p.last_error is not None
+
+
+async def test_arxiv_malformed_xml(arxiv_client):
+    p = ArxivProvider(client=_client(lambda req: httpx.Response(200, text="<<<not xml")), max_attempts=1)
+    assert await p.search_phrase("x", limit=5) == []
+    assert p.last_error is not None
+
+
+async def test_arxiv_fetch_text_downloads_and_extracts(arxiv_client, monkeypatch):
+    # MockTransport serves the PDF bytes; extraction is mocked to avoid a real
+    # pdfplumber parse of arbitrary bytes.
+    def pdf_handler(request):
+        assert "/pdf/" in str(request.url) and request.url.path.endswith(".pdf")
+        return httpx.Response(200, content=b"%PDF-1.4 fake")
+
+    p = ArxivProvider(
+        client=httpx.AsyncClient(transport=httpx.MockTransport(pdf_handler)),
+        max_pdf_fetches=5,
+        max_attempts=1,
+    )
+
+    def fake_extract(path):
+        return {"full_text": "the full text of the paper from the pdf", "title": "t"}
+
+    monkeypatch.setattr("utils.pdf.extract_text_from_pdf", fake_extract)
+    c = Candidate(
+        candidate_id="A1", title="t", url="http://arxiv.org/abs/2101.00001",
+        provider="arXiv", bucket=Bucket.OPEN_ACCESS, text="abstract",
+    )
+    text = await p.fetch_text(c)
+    assert "full text of the paper" in text
+
+
+async def test_arxiv_fetch_budget_caps(arxiv_client, monkeypatch):
+    calls = {"n": 0}
+
+    def fake_extract(path):
+        calls["n"] += 1
+        return {"full_text": "some extracted text", "title": "t"}
+
+    monkeypatch.setattr("utils.pdf.extract_text_from_pdf", fake_extract)
+    p = ArxivProvider(
+        client=arxiv_client, max_pdf_fetches=2, max_attempts=1,
+    )
+    c = Candidate(
+        candidate_id="A1", title="t", url="http://arxiv.org/abs/2101.00001",
+        provider="arXiv", bucket=Bucket.OPEN_ACCESS, text="abstract",
+    )
+    assert await p.fetch_text(c) is not None
+    assert await p.fetch_text(c) is not None
+    assert await p.fetch_text(c) is None  # budget exhausted
+    assert calls["n"] == 2

tests/similarity/test_exclusions.py

@@ -0,0 +1,77 @@
+"""Exclusion-region tests.
+
+The denominator is the word count AFTER exclusions. A paper that is 100%
+bibliography must yield no comparable words - the pipeline reports
+`unavailable`, not 100%. Bibliography, quotes, and the title block are the
+mandatory default exclusions from plan.md §4.1.
+"""
+from similarity.exclusions import find_exclusions
+from similarity.normalize import normalize
+
+
+def _nd(text):
+    return normalize(text)
+
+
+def test_bibliography_section_is_excluded():
+    text = (
+        "This is the introduction with some real content words here. "
+        "\n\nReferences\n"
+        "[1] Smith, J. A very long reference string about something important. "
+        "[2] Jones, K. Another reference that should never be counted as copied."
+    )
+    nd = _nd(text)
+    ex = find_exclusions(nd, exclude_bibliography=True)
+    reasons = {name for name, _, _ in ex.ranges}
+    assert "bibliography" in reasons
+    # every word of the references string is covered by an exclusion
+    ref_text = "Smith J A very long reference string about something important"
+    ref_words = _nd(ref_text).words
+    for w in ref_words:
+        idx = nd.words.index(w)
+        assert ex.covers(idx), f"word {w!r} should be excluded"
+
+
+def test_quoted_text_is_excluded():
+    text = (
+        'The authors state that "this passage is quoted verbatim from another '
+        'work and should not count" and then continue.'
+    )
+    nd = _nd(text)
+    ex = find_exclusions(nd, exclude_quotes=True)
+    reasons = {name for name, _, _ in ex.ranges}
+    assert "quotes" in reasons
+    for i, w in enumerate(nd.words):
+        if w in ("this", "passage", "quoted"):
+            assert ex.covers(i)
+
+
+def test_exclusions_are_optional():
+    text = 'He said "the quick brown fox" and moved on.'
+    nd = _nd(text)
+    with_quotes = find_exclusions(nd, exclude_quotes=True)
+    without_quotes = find_exclusions(nd, exclude_quotes=False)
+    assert "quotes" in {name for name, _, _ in with_quotes.ranges}
+    assert "quotes" not in {name for name, _, _ in without_quotes.ranges}
+
+
+def test_covers_returns_false_outside_exclusions():
+    text = "Clean body text that is definitely not excluded."
+    nd = _nd(text)
+    ex = find_exclusions(nd)
+    for i, w in enumerate(nd.words):
+        if w in ("clean", "body", "definitely"):
+            assert not ex.covers(i)
+
+
+def test_equations_and_captions_are_excluded():
+    text = (
+        "Figure 1: The architecture diagram showing layers and weights. "
+        "x = alpha plus beta times gamma over delta squared. "
+        "Then normal prose resumes again here."
+    )
+    nd = _nd(text)
+    ex = find_exclusions(nd)
+    reasons = {name for name, _, _ in ex.ranges}
+    assert "captions" in reasons
+    assert "equations" in reasons

tests/similarity/test_fingerprint.py

@@ -0,0 +1,89 @@
+"""Fingerprint tests, including the winnowing correctness PROPERTY test.
+
+The plan's whole recall story rests on one guarantee: if two documents share a
+contiguous run of >= w + k - 1 words, their fingerprint sets MUST intersect.
+That guarantee is tested here directly against randomized inputs - if it ever
+fails, every recall claim in plan.md falls with it.
+"""
+import random
+
+import pytest
+
+from similarity.config import settings
+from similarity.fingerprint import fingerprint, kgram_hashes, winnow
+from similarity.normalize import normalize
+
+_K = settings.kgram_size
+_W = settings.window_size
+GUARANTEE_WORDS = _K + _W - 1  # 8 words
+
+
+def _words(text: str) -> list[str]:
+    return normalize(text).words
+
+
+def test_kgram_hashes_length():
+    hashes = kgram_hashes(_words("a b c d e f g"), _K)
+    # 7 words -> 7 - 5 + 1 = 3 k-grams
+    assert len(hashes) == 3
+
+
+def test_kgram_hashes_are_position_independent():
+    # same 5-word tuple anywhere -> same hash
+    h1 = kgram_hashes(_words("the quick brown fox jumps over the lazy"), _K)
+    h2 = kgram_hashes(_words("zzz zzz zzz the quick brown fox jumps end"), _K)
+    shared_tuple = kgram_hashes(_words("the quick brown fox jumps over the lazy"), _K)[0]
+    assert shared_tuple in h1
+    # the tuple "the quick brown fox jumps" occurs at index 3 in the second doc
+    assert shared_tuple in h2
+
+
+def test_kgram_hashes_are_case_insensitive():
+    h1 = kgram_hashes(_words("The Quick Brown Fox Jumps"), _K)
+    h2 = kgram_hashes(_words("the quick brown fox jumps"), _K)
+    assert h1 == h2
+
+
+def test_winnow_deduplicates_adjacent_minima():
+    hashes = [5, 1, 1, 3, 4, 9, 2, 8, 7]
+    selected = winnow(hashes, _W)
+    assert 1 in selected  # the dominant minimum appears
+    assert isinstance(selected, set)
+
+
+def test_fingerprint_identical_text_yields_same_set():
+    a = _words("the quick brown fox jumps over the lazy dog while the sun sets")
+    b = _words("the quick brown fox jumps over the lazy dog while the sun sets")
+    assert fingerprint(a, _K, _W) == fingerprint(b, _K, _W)
+
+
+# ── The winnowing property test ──────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("seed", range(25))
+def test_shared_run_of_w_plus_k_minus_1_words_always_intersects(seed):
+    rng = random.Random(seed)
+    vocab = [f"word{i}" for i in range(200)]
+
+    shared_run = [rng.choice(vocab) for _ in range(GUARANTEE_WORDS)]
+
+    left_pad = [rng.choice(vocab) for _ in range(rng.randint(0, 30))]
+    right_pad = [rng.choice(vocab) for _ in range(rng.randint(0, 30))]
+    doc_a = left_pad + shared_run + right_pad
+
+    left_pad_b = [rng.choice(vocab) for _ in range(rng.randint(0, 30))]
+    right_pad_b = [rng.choice(vocab) for _ in range(rng.randint(0, 30))]
+    doc_b = left_pad_b + shared_run + right_pad_b
+
+    fp_a = fingerprint(doc_a, _K, _W)
+    fp_b = fingerprint(doc_b, _K, _W)
+    assert fp_a & fp_b, (
+        f"seed={seed}: shared run of {GUARANTEE_WORDS} words produced "
+        f"disjoint fingerprints - winnowing guarantee violated"
+    )
+
+
+def test_unrelated_text_yields_disjoint_fingerprints():
+    a = _words("the quick brown fox jumps over the lazy dog on a sunny morning")
+    b = _words("quantum entanglement correlates distant particles mysteriously")
+    assert not (fingerprint(a, _K, _W) & fingerprint(b, _K, _W))

tests/similarity/test_index_writer.py

@@ -0,0 +1,103 @@
+"""Tests for the internal-corpus write path (plan.md §7.4, §16 risk 6).
+
+SIM_INDEX_UPLOADS ships false - indexing is a consent decision, so the write
+path must do nothing by default.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from similarity.index_writer import (
+    COLL_FINGERPRINTS,
+    COLL_TEXTS,
+    ensure_corpus_indexes,
+    index_document,
+    purge_corpus_for_job,
+)
+
+DOC_TEXT = (
+    "Attention based transformer architectures have become the dominant "
+    "approach for sequence modelling tasks in natural language processing "
+    "because they capture long range dependencies far more efficiently than "
+    "recurrent networks while remaining fully parallelisable during training."
+)
+
+
+async def _index(fake_mongo, **kwargs):
+    with patch("similarity.index_writer.settings") as settings:
+        settings.index_uploads = True
+        settings.kgram_size = 5
+        settings.window_size = 4
+        return await index_document(
+            fake_mongo,
+            doc_id="doc_1",
+            job_id="job_1",
+            title="My paper",
+            owner_email="me@example.org",
+            text=DOC_TEXT,
+            **kwargs,
+        )
+
+
+async def test_index_uploads_ships_disabled(fake_mongo):
+    with patch("similarity.index_writer.settings") as settings:
+        settings.index_uploads = False
+        settings.kgram_size = 5
+        settings.window_size = 4
+        ok = await index_document(
+            fake_mongo,
+            doc_id="doc_1", job_id="job_1", title="t", owner_email="e", text=DOC_TEXT,
+        )
+    assert ok is False
+    assert fake_mongo.corpus_fingerprints._docs == []
+    assert fake_mongo.corpus_texts._docs == []
+
+
+async def test_index_document_writes_fingerprint_and_text_rows(fake_mongo):
+    ok = await _index(fake_mongo)
+    assert ok is True
+
+    fp_rows = fake_mongo.corpus_fingerprints._docs
+    assert len(fp_rows) == 1
+    row = fp_rows[0]
+    assert row["doc_id"] == "doc_1"
+    assert row["job_id"] == "job_1"
+    assert row["title"] == "My paper"
+    assert row["owner_email"] == "me@example.org"
+    assert row["word_count"] > 0
+    assert len(row["fingerprints"]) > 0
+    assert row["created_at"] is not None
+
+    text_rows = fake_mongo.corpus_texts._docs
+    assert len(text_rows) == 1
+    assert text_rows[0]["doc_id"] == "doc_1"
+    assert "transformer" in text_rows[0]["normalized_text"]
+    assert "starts" in text_rows[0]["offset_map"]
+
+
+async def test_ensure_corpus_indexes_creates_multikey_on_fingerprints(fake_mongo):
+    await ensure_corpus_indexes(fake_mongo)
+    assert any("fingerprints" in [f[0] if isinstance(f, tuple) else f[0] for f in keys]
+               for keys in fake_mongo.corpus_fingerprints._indexes)
+
+
+async def test_purge_corpus_for_job_removes_rows(fake_mongo):
+    await _index(fake_mongo)
+    assert fake_mongo.corpus_fingerprints._docs
+
+    await purge_corpus_for_job(fake_mongo, job_id="job_1", doc_id="doc_1")
+    assert fake_mongo.corpus_fingerprints._docs == []
+    assert fake_mongo.corpus_texts._docs == []
+
+
+async def test_purge_by_job_id_only(fake_mongo):
+    await _index(fake_mongo)
+    await purge_corpus_for_job(fake_mongo, job_id="job_1", doc_id=None)
+    assert fake_mongo.corpus_fingerprints._docs == []
+
+
+async def test_purge_handles_unknown_doc_id(fake_mongo):
+    await _index(fake_mongo)
+    await purge_corpus_for_job(fake_mongo, job_id="nope", doc_id="nope")
+    assert fake_mongo.corpus_fingerprints._docs  # untouched

tests/similarity/test_matcher.py

@@ -0,0 +1,87 @@
+"""Span-matching tests.
+
+Seeds found via fingerprint intersection must extend into maximal spans with
+exact word and character boundaries, and overlapping seeds must merge into a
+single span rather than two.
+"""
+from similarity.config import settings
+from similarity.matcher import match_documents
+from similarity.normalize import normalize
+
+_K = settings.kgram_size
+_W = settings.window_size
+
+
+def _docs(a, b):
+    return normalize(a), normalize(b)
+
+
+def test_identical_document_matches_entirety():
+    text = (
+        "The quick brown fox jumps over the lazy dog while the sun sets in "
+        "the west and the birds return to their nests to rest at night."
+    )
+    nd_a, nd_b = _docs(text, text)
+    spans = match_documents(nd_a, nd_b, k=_K, w=_W, min_match=settings.min_match_words)
+    assert len(spans) == 1
+    assert spans[0].word_count == len(nd_a.words)
+
+
+def test_disjoint_text_has_no_matches():
+    a = "The quick brown fox jumps over the lazy dog on a sunny morning."
+    b = "Quantum entanglement correlates distant particles in mysterious ways."
+    nd_a, nd_b = _docs(a, b)
+    assert match_documents(nd_a, nd_b, k=_K, w=_W, min_match=settings.min_match_words) == []
+
+
+def test_shared_run_produces_exact_boundaries():
+    shared = "this distinctive phrase is copied verbatim from the original"
+    a = "introduction text that precedes " + shared + " and trailing prose here"
+    b = "different preamble words " + shared + " followed by unrelated content"
+    nd_a, nd_b = _docs(a, b)
+
+    spans = match_documents(nd_a, nd_b, k=_K, w=_W, min_match=settings.min_match_words)
+    assert len(spans) == 1
+    span = spans[0]
+
+    expected_words = len(shared.split())
+    assert span.word_count == expected_words
+
+    # char offsets must slice back to the shared phrase
+    a_shared = nd_a.original_text[span.doc_char_start:span.doc_char_end]
+    assert " ".join(a_shared.split()) == shared
+
+
+def test_overlapping_seeds_merge_into_one_span():
+    # A long shared run yields many overlapping seeds; they must collapse.
+    shared = (
+        "one two three four five six seven eight nine ten eleven twelve "
+        "thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty"
+    )
+    a = "prelude words here then " + shared + " postlude words appear"
+    b = shared
+    nd_a, nd_b = _docs(a, b)
+    spans = match_documents(nd_a, nd_b, k=_K, w=_W, min_match=settings.min_match_words)
+    assert len(spans) == 1
+    assert spans[0].word_count == len(shared.split())
+
+
+def test_short_shared_runs_are_dropped():
+    shared = "a quick phrase"  # 3 words, below min_match
+    a = "intro words before " + shared + " and more words afterwards"
+    b = "totally different lead in " + shared + " then unrelated text here"
+    nd_a, nd_b = _docs(a, b)
+    spans = match_documents(nd_a, nd_b, k=_K, w=_W, min_match=settings.min_match_words)
+    assert spans == []
+
+
+def test_span_carries_source_and_doc_word_indices():
+    shared = "this distinctive phrase is copied verbatim from the original"
+    a = "preface " + shared + " ending"
+    b = "preamble " + shared + " finale"
+    nd_a, nd_b = _docs(a, b)
+    spans = match_documents(nd_a, nd_b, k=_K, w=_W, min_match=settings.min_match_words)
+    span = spans[0]
+    assert span.doc_start_word >= 0
+    assert span.doc_end_word <= len(nd_a.words)
+    assert span.doc_end_word - span.doc_start_word == span.word_count

tests/similarity/test_normalize.py

@@ -0,0 +1,52 @@
+"""Offset-map round-trip tests for similarity.normalize.
+
+The offset map is what makes highlighting land on the right pixels: every word
+in the normalized document must slice the ORIGINAL text back to exactly that
+word. If this ever breaks, spans point at the wrong part of the PDF.
+"""
+from similarity.normalize import normalize
+
+
+def test_every_word_round_trips_into_original_text():
+    text = (
+        "The quick, brown fox! Jumps-over the lazy dog.\n"
+        "  Second line with  numbers 123 and (parens)."
+    )
+    nd = normalize(text)
+    assert len(nd.words) > 0
+    for i, w in enumerate(nd.words):
+        assert nd.ends[i] > nd.starts[i]
+        assert text[nd.starts[i]:nd.ends[i]] == w
+
+
+def test_offsets_are_contiguous_and_monotonic():
+    text = "alpha beta gamma delta epsilon"
+    nd = normalize(text)
+    assert nd.words == ["alpha", "beta", "gamma", "delta", "epsilon"]
+    starts = nd.starts
+    assert starts == sorted(starts)
+    # no overlap: next word starts at or after this word's end
+    for i in range(len(nd.words) - 1):
+        assert nd.ends[i] <= nd.starts[i + 1]
+
+
+def test_punctuation_is_stripped_but_positions_preserved():
+    text = "Hello, world!! (parentheses)  spaced."
+    nd = normalize(text)
+    joined = [text[s:e] for s, e in zip(nd.starts, nd.ends)]
+    # each token is a pure word, no punctuation
+    assert all(tok.isalnum() for tok in joined)
+    # original text unchanged
+    assert text == "Hello, world!! (parentheses)  spaced."
+
+
+def test_empty_text_yields_empty_doc():
+    nd = normalize("")
+    assert nd.words == []
+    assert nd.starts == []
+    assert nd.ends == []
+
+
+def test_whitespace_only_text_yields_empty_doc():
+    nd = normalize("   \n\t   ")
+    assert nd.words == []

tests/similarity/test_pipeline.py

@@ -0,0 +1,205 @@
+"""Pipeline integration tests: normalize -> exclude -> select -> retrieve ->
+verify -> match -> aggregate -> report (plan.md §8).
+
+Uses fake providers (no network) so the orchestration, budget, aggregation,
+and status rules are tested against controlled inputs. The semantic/paraphrase
+pass (Phase `semantic.py`) is tested separately and is asserted to never move
+the headline percentage.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from similarity.corpus.base import Candidate
+from similarity.pipeline import SimilarityPipeline
+from similarity.schema import Bucket
+
+# ~340+ words so the pipeline is not "unavailable".
+DOC_TEXT = (
+    "We study the quantum coherent transport through a one dimensional chain "
+    "coupled to external reservoirs. The conductance exhibits remarkable "
+    "oscillations as a function of the magnetic flux threading the ring. "
+    "We derive an exact expression for the transmission amplitude using the "
+    "Landauer formalism and compare the result against the semiclassical "
+    "approximation. Our analysis shows that interference between the two arms "
+    "of the ring gives rise to periodic resonances whose period is set by the "
+    "flux quantum. We also investigate the role of electron electron "
+    "interactions and disorder in the chain. For weak coupling the system "
+    "behaves as a single channel conductor, whereas strong coupling opens "
+    "additional transverse modes. The temperature dependence of the "
+    "conductance is governed by thermal averaging over the Fermi function. "
+    "We compare our predictions with recent experimental measurements on "
+    "semiconductor rings. The agreement is excellent in the coherent regime. "
+    "We conclude that the mesoscopic ring is an ideal platform for studying "
+    "phase coherent effects and quantum interference at the nanoscale. "
+    "These findings have direct implications for the design of quantum "
+    "interference devices and for understanding decoherence mechanisms. "
+    "The methods we employ combine analytical techniques with numerical "
+    "simulation of the scattering matrix. Our numerical results confirm the "
+    "analytical predictions over a wide range of parameters. "
+    "We finally discuss possible extensions of the model to multi terminal "
+    "geometries and spin dependent transport. "
+    "A key ingredient of our approach is the coupling between the ring and "
+    "the leads which controls the width of the resonances and the visibility "
+    "of the interference fringes. We show that the phase coherence length "
+    "plays a central role in determining whether the system operates in the "
+    "ballistic or the diffusive regime. In the ballistic limit the scattering "
+    "is purely elastic and the conductance is quantized in units of the "
+    "conductance quantum. In the diffusive limit we recover the familiar "
+    "Ohmic behavior with a resistance that scales linearly with the length of "
+    "the wire. We discuss the crossover between these two regimes as the "
+    "temperature is increased and the phase breaking rate grows. "
+    "Our model also accounts for the presence of a weak magnetic impurity "
+    "which introduces an asymmetry between the two propagation directions. "
+    "The impurity acts as a resonant scatterer whose transmission probability "
+    "depends sensitively on the applied flux. This leads to an asymmetric "
+    "lineshape in the conductance as a function of the flux and provides a "
+    "sensitive probe of the impurity potential. "
+    "We compare our findings with a simplified model in which the impurity is "
+    "replaced by an effective delta barrier and show that the two descriptions "
+    "agree to leading order in the coupling strength. "
+    "This work was supported in part by the national research foundation "
+    "under grant number 12345 and by a university fellowship. "
+    "The numerical simulations were performed on a high performance computing "
+    "cluster provided by the department of physics at our institution."
+)
+
+
+class FakeProvider:
+    name = "fake"
+    bucket = Bucket.OPEN_ACCESS
+
+    def __init__(self, phrase_matches: dict[str, list[Candidate]], fail: bool = False,
+                 always: list[Candidate] | None = None):
+        self._phrase_matches = phrase_matches
+        self.fail = fail
+        self.always = always or []
+        self.last_error = None
+        self.queries: list[str] = []
+
+    async def search_phrase(self, phrase, limit):
+        self.queries.append(phrase)
+        if self.fail:
+            self.last_error = "simulated outage"
+            return []
+        if phrase in self._phrase_matches:
+            return self._phrase_matches[phrase]
+        return self.always
+
+    async def fetch_text(self, candidate):
+        return candidate.text
+
+
+def _cand(cid, text, title="A real source", url="https://example.org/paper", bucket=Bucket.OPEN_ACCESS, **kw):
+    return Candidate(
+        candidate_id=cid, title=title, url=url, provider="fake",
+        bucket=bucket, text=text, **kw,
+    )
+
+
+def _pipeline(providers):
+    return SimilarityPipeline(providers=providers)
+
+
+async def test_unavailable_below_minimum_words():
+    pipe = SimilarityPipeline(providers=[FakeProvider({})])
+    report = await pipe.run("only a handful of words here", doc_id="d1")
+    assert report.status == "unavailable"
+
+
+async def test_clean_document_scores_zero():
+    """A document whose phrases match nothing must report 0 and complete."""
+    pipe = _pipeline([FakeProvider({})])
+    report = await pipe.run(DOC_TEXT, doc_id="d1")
+    assert report.status in ("complete", "partial")
+    assert report.overall_percent == 0
+
+
+async def test_full_match_scores_high():
+    """A source containing the whole document => ~100% headline."""
+    prov = FakeProvider({}, always=[_cand("s1", DOC_TEXT)])
+    pipe = _pipeline([prov])
+    report = await pipe.run(DOC_TEXT, doc_id="d1")
+    assert report.overall_percent >= 90
+    assert any(s.bucket is Bucket.OPEN_ACCESS for s in report.sources)
+
+
+async def test_provider_failure_marks_coverage_but_keeps_job_complete():
+    """A CORE outage must yield partial status + recorded failure, not a crash."""
+    failing = FakeProvider({}, fail=True)
+    pipe = _pipeline([failing])
+    report = await pipe.run(DOC_TEXT, doc_id="d1")
+    assert report.status == "partial"
+    assert failing.name in report.coverage.providers_failed
+    assert report.coverage.candidates_retrieved == 0
+
+
+async def test_provider_raising_exception_does_not_crash_pipeline():
+    """An unexpected exception inside a provider must be caught and recorded,
+    exactly like a 429 - the report is partial, never a hard failure."""
+
+    class ExplodingProvider(FakeProvider):
+        async def search_phrase(self, phrase, limit):
+            raise RuntimeError("boom")
+
+        async def fetch_text(self, candidate):
+            return candidate.text
+
+    prov = ExplodingProvider({})
+    pipe = _pipeline([prov])
+    report = await pipe.run(DOC_TEXT, doc_id="d1")
+    assert report.status == "partial"
+    assert prov.name in report.coverage.providers_failed
+
+
+async def test_budget_timeout_yields_partial_with_flag():
+    """If retrieval exceeds the budget, return what we have, flagged."""
+    slow = FakeProvider({})
+
+    async def slow_search(phrase, limit):
+        await asyncio.sleep(0.2)
+        return []
+
+    slow.search_phrase = slow_search
+    pipe = SimilarityPipeline(providers=[slow], budget_seconds=0.05)
+    report = await pipe.run(DOC_TEXT, doc_id="d1")
+    assert report.status == "partial"
+    assert report.coverage.budget_exhausted is True
+
+
+async def test_three_sources_share_one_span_counts_once():
+    """plan.md §13: one span claimed by three sources counts once in the
+    headline, but each source still gets its attributed span."""
+    text_a = DOC_TEXT
+    # three sources that all contain the same span; must count once in headline
+    prov = FakeProvider({}, always=[
+        _cand("s1", text_a),
+        _cand("s2", text_a, bucket=Bucket.PUBLICATION),
+        _cand("s3", text_a, bucket=Bucket.INTERNAL),
+    ])
+    pipe = _pipeline([prov])
+    report = await pipe.run(DOC_TEXT, doc_id="d1")
+    assert report.overall_percent >= 90
+    assert len(report.sources) >= 3
+    # headline cannot exceed 100 no matter how many sources claim the text
+    assert report.overall_percent <= 100
+
+
+async def test_preprint_self_match_candidate_is_dropped():
+    """A candidate that is the uploaded paper's own preprint must be excluded
+    and surfaced in coverage.notes, never scored at ~100%."""
+    prov = FakeProvider({}, always=[
+        _cand(
+            "preprint",
+            DOC_TEXT,
+            title="Quantum coherent transport through a one dimensional chain",
+            url="https://arxiv.org/abs/2101.00001",
+        ),
+    ])
+    pipe = SimilarityPipeline(providers=[prov], doc_title="Quantum coherent transport through a one dimensional chain")
+    report = await pipe.run(DOC_TEXT, doc_id="d1")
+    assert report.overall_percent == 0
+    assert any("self" in n.lower() for n in report.notes)

tests/similarity/test_privacy.py

@@ -0,0 +1,170 @@
+"""Privacy tests for the internal corpus (plan.md §7.4, §16 risk 6).
+
+The privacy boundary is the reason the internal bucket exists at all: another
+researcher's uploaded paper is confidential. The report may show that a match
+exists, the excerpt from the USER'S OWN document, a neutral submission label,
+and the percentage - never the other document's title, URL, authors, full
+text, or a link to it.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from unittest.mock import patch
+
+from similarity.aggregate import Match, compute_percentages
+from similarity.corpus.internal import InternalCorpusProvider, _submitted_label
+from similarity.index_writer import index_document
+from similarity.matcher import match_documents
+from similarity.normalize import normalize
+from similarity.schema import Bucket, MatchSpan
+
+DOC_TEXT = (
+    "The quantum coherent transport through a one dimensional chain exhibits "
+    "remarkable interference phenomena that depend on the magnetic flux "
+    "enclosed by the ring and the coupling strength between the chain and "
+    "the reservoirs. We analyze the conductance oscillations using the "
+    "Landauer formalism and compare our results with recent experiments."
+)
+
+
+async def _index(fake_mongo, *, doc_id="doc_a", text=DOC_TEXT, created_at=None):
+    with patch("similarity.index_writer.settings") as settings:
+        settings.index_uploads = True
+        settings.kgram_size = 5
+        settings.window_size = 4
+        ok = await index_document(
+            fake_mongo,
+            doc_id=doc_id,
+            job_id=f"job_{doc_id}",
+            title="Another researcher's paper",
+            owner_email="someone@else.org",
+            text=text,
+        )
+    if ok and created_at:
+        fake_mongo.corpus_fingerprints._docs[-1]["created_at"] = created_at
+    return ok
+
+
+def test_neutral_label_uses_submission_date():
+    label = _submitted_label(datetime(2026, 3, 14))
+    assert label == "CitationEdge Corpus · submitted 2026-03-14"
+
+
+def test_neutral_label_without_date():
+    assert _submitted_label(None) == "CitationEdge Corpus · submitted unknown-date"
+
+
+async def test_indexed_document_is_found_by_internal_provider(fake_mongo):
+    assert await _index(fake_mongo) is True
+
+    provider = InternalCorpusProvider(fake_mongo, doc_id="doc_new", k=5, w=4)
+    phrase = "quantum coherent transport through a one dimensional chain"
+    candidates = await provider.search_phrase(phrase, limit=10)
+
+    assert len(candidates) == 1
+    assert candidates[0].candidate_id == "doc_a"
+    assert candidates[0].provider == "CitationEdge Corpus"
+    assert candidates[0].bucket is Bucket.INTERNAL
+
+    text = await provider.fetch_text(candidates[0])
+    assert text and "quantum coherent transport" in text
+
+
+async def test_self_match_guard_excludes_own_doc_id(fake_mongo):
+    await _index(fake_mongo, doc_id="doc_a")
+    # Scoring document A against the corpus must not return A itself.
+    provider = InternalCorpusProvider(fake_mongo, doc_id="doc_a", k=5, w=4)
+    phrase = "quantum coherent transport through a one dimensional chain"
+    assert await provider.search_phrase(phrase, limit=10) == []
+
+
+async def test_self_match_guard_excludes_job_id_on_recheck(fake_mongo):
+    await _index(fake_mongo, doc_id="doc_a")
+    # A recheck of the same upload reuses doc_id but the job differs; the
+    # guard must exclude on either axis.
+    provider = InternalCorpusProvider(fake_mongo, doc_id="doc_a", job_id="job_doc_a", k=5, w=4)
+    phrase = "quantum coherent transport through a one dimensional chain"
+    assert await provider.search_phrase(phrase, limit=10) == []
+
+
+async def test_duplicate_submission_scores_approximately_100(fake_mongo):
+    """Exit criterion: index A, then score A against the corpus => ~100%.
+
+    Simulates the paper-submitted-twice case: document A was indexed in an
+    earlier job; a fresh, identical upload (new doc_id/job_id) must find it
+    and verify ~full coverage.
+    """
+    await _index(fake_mongo, doc_id="doc_a")
+    nd_new = normalize(DOC_TEXT)
+
+    # The fresh submission runs the read path with a NEW doc/job identity.
+    provider = InternalCorpusProvider(fake_mongo, doc_id="doc_b", job_id="job_b", k=5, w=4)
+    candidates = await provider.search_phrase(
+        "quantum coherent transport through a one dimensional chain", limit=10
+    )
+    assert len(candidates) == 1 and candidates[0].candidate_id == "doc_a"
+
+    text = await provider.fetch_text(candidates[0])
+    spans = match_documents(nd_new, normalize(text), k=5, w=4, min_match=8)
+    matched = sum(s.word_count for s in spans)
+    assert matched >= int(0.99 * nd_new.word_count)
+
+
+async def test_internal_source_match_carries_only_neutral_label(fake_mongo):
+    """The exit-criterion privacy test: no title/URL/authors/text leaked."""
+    await _index(
+        fake_mongo,
+        created_at=datetime(2026, 3, 14),
+    )
+
+    provider = InternalCorpusProvider(fake_mongo, doc_id="doc_new", k=5, w=4)
+    candidates = await provider.search_phrase(
+        "quantum coherent transport through a one dimensional chain", limit=10
+    )
+    assert candidates, "candidate should be found first"
+
+    text = await provider.fetch_text(candidates[0])
+    nd_doc = normalize(DOC_TEXT)
+    nd_src = normalize(text)
+    spans = match_documents(nd_doc, nd_src, k=5, w=4, min_match=8)
+    assert spans, "a verbatim match should be verified"
+
+    schema_spans = [
+        MatchSpan(
+            doc_start_word=s.doc_start_word,
+            doc_end_word=s.doc_end_word,
+            doc_char_start=s.doc_char_start,
+            doc_char_end=s.doc_char_end,
+            word_count=s.word_count,
+            excerpt=s.excerpt,
+            source_excerpt=s.source_excerpt,
+            source_index=1,
+        )
+        for s in spans
+    ]
+
+    matched_words = sum(s.word_count for s in spans)
+    result = compute_percentages(
+        nd_doc.word_count,
+        [Match(source_index=1, bucket=Bucket.INTERNAL, trust=3, doc_start=0, doc_end=nd_doc.word_count)],
+    )
+    source = provider.build_source_match(
+        candidates[0],
+        source_index=1,
+        matched_words=matched_words,
+        percent=result.overall_percent,
+        spans=schema_spans,
+    )
+
+    # The neutral label is the ONLY thing about the other document allowed out.
+    assert source.title == "CitationEdge Corpus · submitted 2026-03-14"
+    assert source.url == "CitationEdge Corpus · submitted 2026-03-14"
+    assert source.authors == []
+    assert source.doi is None
+    assert source.provider == "CitationEdge Corpus"
+    assert "someone@else.org" not in source.model_dump_json()
+    assert "Another researcher" not in source.model_dump_json()
+    # No text from the other document may escape in a span excerpt.
+    for span in source.spans:
+        assert span.source_excerpt == ""

tests/similarity/test_selector.py

@@ -0,0 +1,100 @@
+"""Query-phrase selection tests.
+
+Phrases must be distinctive, spread across the document, and never boilerplate
+or all-stopword. The selected phrases are what get sent to CORE as
+`fullText:"..."` queries, so their quality bounds the recall of stage 1.
+"""
+from similarity.exclusions import find_exclusions
+from similarity.normalize import normalize
+from similarity.selector import select_phrases
+
+
+def _doc(text, **kw):
+    nd = normalize(text)
+    return nd, find_exclusions(nd, **kw)
+
+
+def test_selects_distinctive_phrases():
+    text = (
+        "In this paper we propose a novel architecture for question answering. "
+        "The encoder stack uses multi head attention mechanisms and positional "
+        "encodings. We evaluate against the standard benchmark suite and report "
+        "improvements across every task category. The decoder combines token "
+        "probabilities with beam search decoding strategies. Training uses a "
+        "masked language objective over a large unlabeled corpus sample."
+    )
+    nd, ex = _doc(text)
+    phrases = select_phrases(nd, ex, max_queries=4)
+    assert len(phrases) > 0
+    assert all(len(p.split()) >= 8 for p in phrases)
+
+
+def test_boilerplate_phrases_are_never_selected():
+    text = (
+        "In this paper we propose a method to improve results significantly. "
+        "The rest of this paper is organized as follows section two describes "
+        "related work and section three presents our approach in detail. "
+        "We then discuss experimental findings and conclude with a summary of "
+        "contributions and directions for future work."
+    )
+    nd, ex = _doc(text)
+    phrases = select_phrases(nd, ex, max_queries=4)
+    joined = " | ".join(p.lower() for p in phrases)
+    assert "rest of this paper is organized" not in joined
+    assert "in this paper we propose" not in joined
+
+
+def test_all_stopword_phrases_are_never_selected():
+    text = (
+        "This is a test of the system and the way that it works. "
+        "The and of in to for with at by on as an or if then so but. "
+        "Quantum entanglement produces correlated measurement outcomes across "
+        "spatially separated laboratories. The experiment confirms the "
+        "violation of a bell inequality under strict locality conditions."
+    )
+    nd, ex = _doc(text)
+    phrases = select_phrases(nd, ex, max_queries=4)
+    for p in phrases:
+        tokens = p.lower().split()
+        assert len([t for t in tokens if t in _ALL_STOPWORDS]) < len(tokens)
+
+
+_ALL_STOPWORDS = {
+    "a", "an", "the", "of", "on", "in", "to", "for", "and", "or", "is", "are",
+    "was", "were", "be", "been", "being", "it", "its", "this", "that", "these",
+    "those", "with", "from", "by", "as", "at", "into", "through", "between",
+}
+
+
+def test_phrases_are_spread_across_the_document():
+    text = (
+        "The first section introduces the topic and motivates the study with "
+        "concrete examples drawn from recent literature surveys. "
+        "The second section develops the mathematical framework and derives "
+        "the main theoretical results with complete proofs. "
+        "The third section describes the experimental setup and the datasets "
+        "used to validate the proposed methodology thoroughly. "
+        "The fourth section reports the empirical results and compares them "
+        "against several strong baseline methods from the literature."
+    )
+    nd, ex = _doc(text)
+    phrases = select_phrases(nd, ex, max_queries=4)
+    positions = [_phrase_start(nd, p) for p in phrases]
+    assert max(positions) - min(positions) > len(nd.words) // 2
+
+
+def _phrase_start(nd, phrase):
+    """Locate the real start index of `phrase` inside the document."""
+    tokens = phrase.split()
+    n = len(tokens)
+    for i in range(len(nd.words) - n + 1):
+        if nd.words[i].lower() == tokens[0].lower() and (
+            " ".join(nd.words[i:i + n]).lower() == phrase.lower()
+        ):
+            return i
+    return -1
+
+
+def test_no_candidates_in_short_document():
+    nd, ex = _doc("just a few words here")
+    assert select_phrases(nd, ex, max_queries=4) == []

tests/similarity/test_self_match_guard.py

@@ -0,0 +1,94 @@
+"""Preprint self-match guard tests (plan.md §16 risk 2).
+
+A published paper matches its own arXiv preprint at ~100% and would produce
+a terrifying, meaningless score - the most likely first bug report. After
+retrieval and before scoring, drop any candidate whose normalized title has
+>= 0.9 token overlap with the uploaded paper's title, or whose DOI matches,
+or whose author set overlaps by >= 50% while similarity is > 60%.
+"""
+
+from __future__ import annotations
+
+from similarity.pipeline import (
+    _token_overlap,
+    _author_overlap,
+    is_preprint_self_match,
+)
+
+
+def test_token_overlap_perfect():
+    a = "attention is all you need"
+    b = "attention is all you need"
+    assert _token_overlap(a, b) == 1.0
+
+
+def test_token_overlap_partial():
+    a = "attention is all you need a paper"
+    b = "attention is all you need"
+    assert _token_overlap(a, b) >= 0.9
+
+
+def test_token_overlap_low():
+    a = "quantum computing circuits"
+    b = "attention is all you need"
+    assert _token_overlap(a, b) < 0.5
+
+
+def test_author_overlap_half():
+    assert _author_overlap(["a", "b"], ["a", "b"]) == 1.0
+    assert _author_overlap(["a", "b", "c"], ["a", "b", "x"]) >= 0.5
+    assert _author_overlap(["a"], ["x", "y"]) == 0.0
+
+
+def test_self_match_by_title():
+    assert is_preprint_self_match(
+        doc_title="Attention Is All You Need",
+        cand_title="Attention is all you need",
+        cand_doi=None,
+        cand_authors=[],
+        similarity=95,
+    ) is True
+
+
+def test_self_match_by_doi():
+    assert is_preprint_self_match(
+        doc_title="My Paper",
+        cand_title="My Paper (preprint)",
+        cand_doi="10.1007/x",
+        cand_authors=[],
+        similarity=95,
+        doc_doi="10.1007/x",
+    ) is True
+
+
+def test_self_match_by_authors_at_high_similarity():
+    # same author team + high similarity + partially overlapping title
+    assert is_preprint_self_match(
+        doc_title="Improving speech recognition",
+        cand_title="Improving speech recognition with deep models",
+        cand_doi=None,
+        cand_authors=["Alice", "Bob", "Carol"],
+        similarity=95,
+        doc_authors=["Alice", "Bob"],
+    ) is True
+
+
+def test_not_self_match_distinct_paper():
+    assert is_preprint_self_match(
+        doc_title="Quantum coherent transport",
+        cand_title="A completely different paper about gardens",
+        cand_doi="10.1016/garden",
+        cand_authors=["Zoe"],
+        similarity=2,
+    ) is False
+
+
+def test_not_self_match_shared_author_low_similarity():
+    # same authors but low similarity => it is a genuine different work
+    assert is_preprint_self_match(
+        doc_title="Quantum rings",
+        cand_title="Another work by the same lab",
+        cand_doi=None,
+        cand_authors=["Alice", "Bob"],
+        similarity=10,
+    ) is False

tests/similarity/test_semantic.py

@@ -0,0 +1,84 @@
+"""Semantic (paraphrase) pass tests (plan.md §4/§5, prompt Phase 3).
+
+The headline percentage is VERBATIM ONLY. The paraphrase pass may flag spans
+for human review (`paraphrase_flags`), but it must NEVER move
+`overall_percent` - otherwise a fuzzy embedding could turn 2% into 80% and
+the whole corpus claim collapses. This is asserted directly.
+"""
+
+from __future__ import annotations
+
+from similarity.semantic import semantic_pass
+from similarity.schema import Bucket, MatchSpan, SimilarityReport, Coverage, SourceMatch
+
+SOURCE_EXCERPT = "the quick brown fox jumps over the lazy dog near the river"
+DOC_EXCERPT = "the swift brown fox leaps above the idle canine beside the stream"
+
+
+def _report(excerpts: list[str] | None = None) -> SimilarityReport:
+    spans = [
+        MatchSpan(
+            doc_start_word=0, doc_end_word=10, doc_char_start=0, doc_char_end=40,
+            word_count=10, excerpt=DOC_EXCERPT, source_excerpt=SOURCE_EXCERPT,
+            source_index=1,
+        )
+    ]
+    return SimilarityReport(
+        overall_percent=50,
+        bucket_percents={Bucket.OPEN_ACCESS: 50},
+        sources=[
+            SourceMatch(
+                source_index=1,
+                bucket=Bucket.OPEN_ACCESS,
+                title="A real source",
+                url="https://example.org/paper",
+                provider="fake",
+                display_label="https://example.org/paper",
+                matched_words=10,
+                percent=10,
+                spans=spans,
+            )
+        ],
+        coverage=Coverage(
+            total_words_compared=100, words_excluded=0,
+            providers_queried=["CORE"], phrases_queried=24,
+            candidates_retrieved=5, candidates_verified=5,
+            internal_corpus_size=10, min_match_words=8,
+        ),
+        status="complete",
+    )
+
+
+def test_semantic_pass_never_changes_headline_percent():
+    base = _report()
+
+    class FakeEmbedder:
+        def embed(self, text):
+            # one dimension so cosine is trivially 1.0 for any pair
+            return [1.0]
+
+        def cosine_similarity(self, a, b):
+            return 1.0
+
+    out = semantic_pass(base, doc_text=DOC_EXCERPT, embedder=FakeEmbedder())
+    # paraphrase is real (flagged for review) ...
+    assert len(out.paraphrase_flags) == 1
+    assert out.paraphrase_flags[0].source_index == 1
+    # ... but the headline NEVER moved.
+    assert out.overall_percent == base.overall_percent == 50
+    assert out.bucket_percents == base.bucket_percents
+
+
+def test_semantic_pass_disabled_by_config(monkeypatch):
+    monkeypatch.setattr("similarity.semantic.settings.enable_paraphrase", False)
+    base = _report()
+    out = semantic_pass(base, doc_text=DOC_EXCERPT, embedder=None)
+    assert out == base
+    assert out.paraphrase_flags == []
+
+
+def test_semantic_pass_returns_empty_when_no_embedder():
+    base = _report()
+    out = semantic_pass(base, doc_text=DOC_EXCERPT, embedder=None)
+    # without an embedder there is nothing to compare; no flags, no score change
+    assert out.paraphrase_flags == []

tests/test_agents.py

@@ -5,6 +5,7 @@ from agents.scoring_agent import ScoringAgent
 from agents.argumentation_agent import ArgumentationAgent
 from agents.citation_relevance_agent import CitationRelevanceAgent
 from agents.novelty_agent import NoveltyAgent
+from agents.similarity_agent import SimilarityAgent
 
 
 @pytest.mark.asyncio
@@ -220,3 +221,16 @@ async def test_novelty_survives_llm_and_search_failure(agent_ctx, mock_neo4j, mo
     result = await agent.run(agent_ctx)
     assert result.status == AgentStatus.COMPLETED
     assert result.data["verdict"] == "UNKNOWN"
+
+
+@pytest.mark.asyncio
+async def test_similarity_agent_unavailable_on_short_text(agent_ctx, mock_neo4j):
+    """SimilarityAgent must return an 'unavailable' report, not crash, on short text."""
+    mock_neo4j.run.return_value = [
+        {"text": "a short paragraph with very few words in it"}
+    ]
+    agent = SimilarityAgent()
+    result = await agent.run(agent_ctx)
+    assert result.status == AgentStatus.COMPLETED
+    assert result.data.get("status") == "unavailable"
+    assert result.data.get("overall_percent") == 0