Deploy MSC Aging Corpus research API: REST /research, OLS reruns, chat UI

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY msc_corpus ./msc_corpus
+COPY app.py ./app.py
+
+ENV CORPUS_REVISION=corpus-v2026.06.6
+ENV CORPUS_CACHE_DIR=/tmp/msc_corpus_cache
+
+EXPOSE 7860
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,10 +1,58 @@
 ---
-title: Msc Aging Corpus Api
-emoji: 🌖
-colorFrom: pink
+title: MSC Aging Corpus API
+emoji: 🧬
+colorFrom: blue
 colorTo: green
 sdk: docker
+app_port: 7860
 pinned: false
+license: cc-by-4.0
+short_description: REST + chat research API for MSC aging transcriptome corpus
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# MSC Aging Corpus Research API
+
+**Single live connection point** for agents and researchers querying the
+[MSC Aging Agent Corpus](https://huggingface.co/datasets/S4MPL3BI4S/msc-aging-agent-corpus).
+
+Curated by **Dr. James Utley, PhD** · **Syndicate Laboratories**
+
+## Agent URL (use this)
+
+**API base:** `https://S4MPL3BI4S-msc-aging-corpus-api.hf.space`
+
+| Endpoint | Purpose |
+| --- | --- |
+| `GET /` | Service info |
+| `GET /docs` | OpenAPI — **agents start here** |
+| `POST /research` | Multi-step hypothesis workflow |
+| `GET /search?gene=NDRG1` | Cross-dataset biomarker hits |
+| `GET /rerun?gene=NDRG1&dataset_id=GSE39540` | Fresh OLS statistics |
+| `GET /expression?gene=NDRG1&dataset_id=GSE39540` | Sample-level data |
+| `GET /manifest` | Cohort scope |
+| `GET /cite?gene=NDRG1&datasets=GSE39540` | APA citations |
+| `/chat` | Structured chat UI |
+
+## Example (curl)
+
+```bash
+curl -X POST "https://S4MPL3BI4S-msc-aging-corpus-api.hf.space/research" \
+  -H "Content-Type: application/json" \
+  -d '{"gene":"NDRG1","dataset_id":"GSE39540","species":"Homo sapiens","actions":["search","rerun","cite"]}'
+```
+
+## Example (Python)
+
+```python
+import requests
+
+API = "https://S4MPL3BI4S-msc-aging-corpus-api.hf.space"
+r = requests.post(f"{API}/research", json={
+    "gene": "NDRG1",
+    "dataset_id": "GSE39540",
+    "actions": ["search", "rerun", "cite"],
+})
+print(r.json())
+```
+
+Data is pulled from Dataset `S4MPL3BI4S/msc-aging-agent-corpus` @ `corpus-v2026.06.6`.

app.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+"""FastAPI research API for the MSC Aging Agent Corpus (Hugging Face Space)."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from contextlib import asynccontextmanager
+from typing import Any
+
+import gradio as gr
+import pandas as pd
+from fastapi import FastAPI, HTTPException, Query
+from pydantic import BaseModel, Field
+
+from msc_corpus import connect
+
+DEFAULT_REVISION = os.environ.get("CORPUS_REVISION", "corpus-v2026.06.6")
+CACHE_DIR = os.environ.get("CORPUS_CACHE_DIR", "/tmp/msc_corpus_cache")
+
+corpus = None
+
+
+@asynccontextmanager
+async def lifespan(_app: FastAPI):
+    global corpus
+    corpus = connect(remote_only=True, revision=DEFAULT_REVISION, cache_dir=CACHE_DIR)
+    yield
+
+
+app = FastAPI(
+    title="MSC Aging Corpus Research API",
+    description=(
+        "Query the Syndicate Laboratories MSC Aging Agent Corpus: biomarker search, "
+        "sample-level expression, OLS statistical reruns, and APA citations. "
+        "Data source: Hugging Face Dataset S4MPL3BI4S/msc-aging-agent-corpus."
+    ),
+    version="1.0.0",
+    lifespan=lifespan,
+)
+
+
+class ResearchRequest(BaseModel):
+    gene: str = Field(..., description="Gene symbol, e.g. NDRG1")
+    dataset_id: str | None = Field(None, description="Optional GEO accession, e.g. GSE39540")
+    species: str | None = Field(None, description="Optional species filter, e.g. Homo sapiens")
+    fdr_only: bool = False
+    actions: list[str] = Field(
+        default=["search", "rerun", "cite"],
+        description="Steps: search, expression, rerun, mechanism, cite",
+    )
+
+
+def _require_corpus():
+    if corpus is None:
+        raise HTTPException(status_code=503, detail="Corpus not loaded yet.")
+    return corpus
+
+
+def _records(frame: pd.DataFrame, limit: int = 500) -> dict[str, Any]:
+    if frame.empty:
+        return {"count": 0, "rows": []}
+    trimmed = frame.head(limit)
+    return {
+        "count": int(len(frame)),
+        "truncated": len(frame) > limit,
+        "rows": json.loads(trimmed.to_json(orient="records")),
+    }
+
+
+@app.get("/")
+def root() -> dict[str, str]:
+    return {
+        "service": "MSC Aging Corpus Research API",
+        "curator": "James Utley, PhD · Syndicate Laboratories",
+        "dataset": "S4MPL3BI4S/msc-aging-agent-corpus",
+        "revision": DEFAULT_REVISION,
+        "openapi_docs": "/docs",
+        "chat_ui": "/chat",
+        "research_endpoint": "POST /research",
+    }
+
+
+@app.get("/health")
+def health() -> dict[str, str]:
+    _require_corpus()
+    return {"status": "ok", "revision": DEFAULT_REVISION}
+
+
+@app.get("/connect")
+def connection_manifest() -> dict[str, Any]:
+    return _require_corpus().connection_info()
+
+
+@app.get("/manifest")
+def manifest(
+    species: str | None = None,
+    limit: int = Query(100, ge=1, le=500),
+) -> dict[str, Any]:
+    frame = _require_corpus().manifest()
+    if species:
+        frame = frame.loc[frame["species"] == species]
+    return _records(frame, limit=limit)
+
+
+@app.get("/search")
+def search_gene(
+    gene: str,
+    species: str | None = None,
+    dataset_id: str | None = None,
+    fdr_only: bool = False,
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict[str, Any]:
+    frame = _require_corpus().search_gene(
+        gene,
+        species=species,
+        dataset_id=dataset_id,
+        fdr_only=fdr_only,
+    )
+    return _records(frame, limit=limit)
+
+
+@app.get("/expression")
+def expression(
+    gene: str,
+    dataset_id: str,
+    limit: int = Query(500, ge=1, le=5000),
+) -> dict[str, Any]:
+    try:
+        frame = _require_corpus().expression(gene, dataset_id)
+    except Exception as exc:  # noqa: BLE001
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+    return _records(frame, limit=limit)
+
+
+@app.get("/rerun")
+def rerun_model(gene: str, dataset_id: str) -> dict[str, Any]:
+    try:
+        result = _require_corpus().rerun_model(gene, dataset_id)
+    except ValueError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+    result.pop("summary", None)
+    return result
+
+
+@app.get("/cite")
+def cite(gene: str, datasets: str = Query(..., description="Comma-separated GSE IDs")) -> dict[str, str]:
+    used = [item.strip() for item in datasets.split(",") if item.strip()]
+    return _require_corpus().cite_gene(gene, datasets_used=used)
+
+
+@app.post("/research")
+def research(body: ResearchRequest) -> dict[str, Any]:
+    try:
+        return _require_corpus().research(
+            body.gene,
+            dataset_id=body.dataset_id,
+            species=body.species,
+            actions=body.actions,
+            fdr_only=body.fdr_only,
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+
+def _parse_chat(message: str) -> tuple[str, dict[str, Any] | None]:
+    text = message.strip()
+    lower = text.lower()
+
+    if lower in {"help", "?"}:
+        return (
+            "Commands:\n"
+            "- `search NDRG1` or `search NDRG1 human`\n"
+            "- `rerun NDRG1 GSE39540`\n"
+            "- `research NDRG1 GSE39540` (search + rerun + cite)\n"
+            "- `manifest` or `manifest human`\n"
+            "Agents should prefer REST: POST /research or GET /docs",
+            None,
+        )
+
+    if lower.startswith("manifest"):
+        species = "Homo sapiens" if "human" in lower else None
+        payload = manifest(species=species)
+        return json.dumps(payload, indent=2), None
+
+    rerun_match = re.match(r"rerun\s+(\S+)\s+(GSE\d+)", text, re.I)
+    if rerun_match:
+        gene, gse = rerun_match.groups()
+        return json.dumps(rerun_model(gene, gse), indent=2), None
+
+    research_match = re.match(r"research\s+(\S+)(?:\s+(GSE\d+))?", text, re.I)
+    if research_match:
+        gene, gse = research_match.groups()
+        body = ResearchRequest(gene=gene, dataset_id=gse)
+        return json.dumps(research(body), indent=2), None
+
+    search_match = re.match(r"search\s+(\S+)(?:\s+(human|mouse|rat))?", text, re.I)
+    if search_match:
+        gene, sp = search_match.groups()
+        species = {"human": "Homo sapiens", "mouse": "Mus musculus", "rat": "Rattus norvegicus"}.get(
+            (sp or "").lower()
+        )
+        return json.dumps(search_gene(gene, species=species), indent=2), None
+
+    return (
+        "I did not understand that. Try `help`, `search NDRG1`, `rerun NDRG1 GSE39540`, "
+        "or `research NDRG1 GSE39540`. For full control use /docs.",
+        None,
+    )
+
+
+def chat_fn(message: str, history: list[dict[str, str]]) -> str:
+    del history
+    reply, _ = _parse_chat(message)
+    return reply
+
+
+demo = gr.ChatInterface(
+    fn=chat_fn,
+    title="MSC Aging Corpus Research Chat",
+    description=(
+        "Structured queries against the MSC Aging Agent Corpus. "
+        "For programmatic access open **/docs** (REST API)."
+    ),
+    examples=[
+        "help",
+        "search NDRG1 human",
+        "rerun NDRG1 GSE39540",
+        "research NDRG1 GSE39540",
+        "manifest human",
+    ],
+)
+
+app = gr.mount_gradio_app(app, demo, path="/chat")

msc_corpus/__init__.py

@@ -0,0 +1,5 @@
+"""Single-connection Python client for the MSC Aging Agent Corpus."""
+
+from msc_corpus.client import MSCAgingCorpus, connect
+
+__all__ = ["MSCAgingCorpus", "connect"]

msc_corpus/__main__.py

@@ -0,0 +1,70 @@
+"""CLI entry point: uv run python -m msc_corpus search NDRG1 --dataset GSE39540 --rerun"""
+
+from __future__ import annotations
+
+import argparse
+import json
+
+from msc_corpus import connect
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="MSC Aging Agent Corpus — single-connection research CLI",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    sub.add_parser("info", help="Print connection manifest")
+
+    search = sub.add_parser("search", help="Search biomarker hits for a gene")
+    search.add_argument("gene")
+    search.add_argument("--species", default=None)
+    search.add_argument("--dataset", default=None)
+    search.add_argument("--fdr-only", action="store_true")
+
+    rerun = sub.add_parser("rerun", help="Rerun linear model for a gene in one GSE")
+    rerun.add_argument("gene")
+    rerun.add_argument("dataset")
+
+    expr = sub.add_parser("expression", help="Fetch sample-level expression rows")
+    expr.add_argument("gene")
+    expr.add_argument("dataset")
+
+    args = parser.parse_args()
+    corpus = connect()
+
+    if args.command == "info":
+        print(json.dumps(corpus.connection_info(), indent=2))
+        return
+
+    if args.command == "search":
+        hits = corpus.search_gene(
+            args.gene,
+            species=args.species,
+            dataset_id=args.dataset,
+            fdr_only=args.fdr_only,
+        )
+        print(hits.to_string(index=False) if not hits.empty else "No hits.")
+        return
+
+    if args.command == "rerun":
+        result = corpus.rerun_model(args.gene, args.dataset)
+        print(result["summary"])
+        print(json.dumps(
+            {
+                "coefficients": result["coefficients"],
+                "p_values": result["p_values"],
+                "r_squared": result["r_squared"],
+                "n_samples": result["n_samples"],
+            },
+            indent=2,
+        ))
+        return
+
+    if args.command == "expression":
+        frame = corpus.expression(args.gene, args.dataset)
+        print(frame.to_string(index=False) if not frame.empty else "No expression rows.")
+
+
+if __name__ == "__main__":
+    main()

msc_corpus/client.py

@@ -0,0 +1,290 @@
+"""Research client for the MSC Aging Agent Corpus (Hugging Face Dataset)."""
+
+from __future__ import annotations
+
+import gzip
+import io
+import json
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+DEFAULT_REPO = "S4MPL3BI4S/msc-aging-agent-corpus"
+DEFAULT_REVISION = "corpus-v2026.06.6"
+
+
+def _repo_root() -> Path:
+    return Path(__file__).resolve().parents[1]
+
+
+def connect(
+    *,
+    revision: str = DEFAULT_REVISION,
+    repo_id: str = DEFAULT_REPO,
+    local_root: Path | None = None,
+    cache_dir: str | None = None,
+    remote_only: bool = False,
+) -> MSCAgingCorpus:
+    """Connect to the corpus from Hugging Face or a local clone."""
+    return MSCAgingCorpus(
+        revision=revision,
+        repo_id=repo_id,
+        local_root=local_root,
+        cache_dir=cache_dir,
+        remote_only=remote_only,
+    )
+
+
+class MSCAgingCorpus:
+    """Agent-first research interface to screened MSC aging transcriptome data."""
+
+    def __init__(
+        self,
+        *,
+        revision: str = DEFAULT_REVISION,
+        repo_id: str = DEFAULT_REPO,
+        local_root: Path | None = None,
+        cache_dir: str | None = None,
+        remote_only: bool = False,
+    ) -> None:
+        self.revision = revision
+        self.repo_id = repo_id
+        self.remote_only = remote_only
+        self.local_root = None if remote_only else (local_root or _repo_root())
+        self.cache_dir = cache_dir
+        self._frames: dict[str, pd.DataFrame] = {}
+        self._byte_cache: dict[str, bytes] = {}
+
+    def connection_info(self) -> dict[str, Any]:
+        return json.loads(self._read_text("reference/AGENT_CONNECTION.json"))
+
+    def provenance(self) -> dict[str, Any]:
+        return json.loads(self._read_text("datasets/CORPUS_PROVENANCE.json"))
+
+    def manifest(self) -> pd.DataFrame:
+        return self._load_csv("datasets/dataset_manifest.csv")
+
+    def biomarkers(self) -> pd.DataFrame:
+        return self._load_csv("datasets/aging_biomarker_candidates.csv")
+
+    def search_gene(
+        self,
+        gene_symbol: str,
+        *,
+        species: str | None = None,
+        fdr_only: bool = False,
+        dataset_id: str | None = None,
+    ) -> pd.DataFrame:
+        """Cross-dataset biomarker hits for a gene."""
+        df = self.biomarkers()
+        mask = df["gene_symbol"].str.upper() == gene_symbol.upper()
+        if species:
+            mask &= df["species"] == species
+        if dataset_id:
+            mask &= df["dataset_id"] == dataset_id
+        if fdr_only:
+            mask &= df["fdr_significant"].astype(str).str.upper().eq("TRUE")
+        return df.loc[mask].sort_values(["q_value", "p_value"], na_position="last")
+
+    def screened_hits(
+        self,
+        gene_symbol: str,
+        *,
+        dataset_id: str | None = None,
+        fdr_only: bool = False,
+    ) -> pd.DataFrame:
+        """Feature-level screened associations, optionally scoped to one GSE."""
+        if dataset_id:
+            datasets = [dataset_id]
+        else:
+            datasets = sorted(self.search_gene(gene_symbol)["dataset_id"].unique())
+        frames: list[pd.DataFrame] = []
+        for gse in datasets:
+            path = f"datasets/{gse}/screened_gene_associations.csv"
+            try:
+                df = self._load_csv(path)
+            except FileNotFoundError:
+                continue
+            mask = df["gene_symbol"].str.upper() == gene_symbol.upper()
+            if fdr_only and "fdr_significant" in df.columns:
+                mask &= df["fdr_significant"].astype(str).str.upper().eq("TRUE")
+            hit = df.loc[mask]
+            if not hit.empty:
+                frames.append(hit)
+        if not frames:
+            return pd.DataFrame()
+        return pd.concat(frames, ignore_index=True)
+
+    def expression(self, gene_symbol: str, dataset_id: str) -> pd.DataFrame:
+        """Sample-level expression plus metadata for modeling."""
+        path = f"datasets/{dataset_id}/expression_screened_long.csv.gz"
+        df = self._load_csv(path)
+        return df.loc[df["gene_symbol"].str.upper() == gene_symbol.upper()].copy()
+
+    def rerun_model(self, gene_symbol: str, dataset_id: str) -> dict[str, Any]:
+        """Rerun the documented linear model for hypothesis testing."""
+        import statsmodels.formula.api as smf
+
+        manifest_row = self.manifest().loc[self.manifest()["dataset_id"] == dataset_id]
+        if manifest_row.empty:
+            raise ValueError(f"Unknown dataset_id: {dataset_id}")
+        formula = manifest_row.iloc[0]["model_formula"]
+        if not str(formula).startswith("~"):
+            raise ValueError(f"Unexpected model_formula for {dataset_id}: {formula}")
+
+        expr = self.expression(gene_symbol, dataset_id)
+        if expr.empty:
+            raise ValueError(f"No expression rows for {gene_symbol} in {dataset_id}")
+
+        if "feature_id" in expr.columns:
+            best_feature = (
+                expr.groupby("feature_id")["q_value"].min().sort_values().index[0]
+            )
+            expr = expr.loc[expr["feature_id"] == best_feature].copy()
+
+        model_formula = f"expression_value {formula}"
+        model = smf.ols(model_formula, data=expr).fit()
+        coef = model.params.to_dict()
+        pvalues = model.pvalues.to_dict()
+        return {
+            "dataset_id": dataset_id,
+            "gene_symbol": gene_symbol,
+            "model_formula": model_formula,
+            "n_samples": int(model.nobs),
+            "r_squared": float(model.rsquared),
+            "adj_r_squared": float(model.rsquared_adj),
+            "coefficients": coef,
+            "p_values": pvalues,
+            "summary": str(model.summary()),
+        }
+
+    def clinical_studies(self, condition: str | None = None) -> pd.DataFrame:
+        df = self._load_csv("datasets/clinical_evidence/dvc_stem_study_manifest.csv")
+        if condition:
+            return df.loc[df["condition_category"].str.contains(condition, case=False, na=False)]
+        return df
+
+    def mechanism_hits(self, gene_symbol: str | None = None) -> pd.DataFrame:
+        df = self._load_csv("datasets/mechanistic_associations/secretome_transcriptome_hits.csv")
+        if gene_symbol:
+            return df.loc[df["gene_symbol"].str.upper() == gene_symbol.upper()]
+        return df
+
+    def cite_corpus(self) -> str:
+        return self.provenance().get("corpus_citation_apa_hf", "")
+
+    def cite_gene(self, gene_symbol: str, datasets_used: list[str]) -> dict[str, str]:
+        out = {"corpus": self.cite_corpus()}
+        try:
+            citations = json.loads(self._read_text("reference/citations_apa.json"))
+            out["corpus"] = citations["corpus_citation"]["hf_distribution"]
+            geo = citations.get("transcriptome_geo", {})
+            for gse in datasets_used:
+                if gse in geo:
+                    out[gse] = geo[gse]["apa"]
+        except Exception:
+            for gse in datasets_used:
+                out[gse] = f"https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc={gse}"
+        return out
+
+    def research(
+        self,
+        gene_symbol: str,
+        *,
+        dataset_id: str | None = None,
+        species: str | None = None,
+        actions: list[str] | None = None,
+        fdr_only: bool = False,
+    ) -> dict[str, Any]:
+        """Run a multi-step research workflow (search, expression, rerun, cite)."""
+        steps = actions or ["search", "rerun", "cite"]
+        result: dict[str, Any] = {"gene_symbol": gene_symbol, "actions": steps}
+        datasets_used: list[str] = []
+
+        if "search" in steps:
+            search = self.search_gene(
+                gene_symbol,
+                species=species,
+                dataset_id=dataset_id,
+                fdr_only=fdr_only,
+            )
+            result["search"] = _frame_payload(search)
+            datasets_used = sorted(search["dataset_id"].unique()) if not search.empty else []
+
+        target_dataset = dataset_id
+        if not target_dataset and datasets_used:
+            target_dataset = datasets_used[0]
+
+        if "expression" in steps:
+            if not target_dataset:
+                result["expression"] = {"error": "No dataset_id and no search hits."}
+            else:
+                result["expression"] = _frame_payload(
+                    self.expression(gene_symbol, target_dataset)
+                )
+
+        if "rerun" in steps:
+            if not target_dataset:
+                result["rerun"] = {"error": "No dataset_id and no search hits."}
+            else:
+                rerun = self.rerun_model(gene_symbol, target_dataset)
+                rerun.pop("summary", None)
+                result["rerun"] = rerun
+                if target_dataset not in datasets_used:
+                    datasets_used.append(target_dataset)
+
+        if "mechanism" in steps:
+            result["mechanism"] = _frame_payload(self.mechanism_hits(gene_symbol))
+
+        if "cite" in steps:
+            result["citations"] = self.cite_gene(gene_symbol, datasets_used=datasets_used)
+
+        return result
+
+    def _load_csv(self, rel_path: str) -> pd.DataFrame:
+        if rel_path in self._frames:
+            return self._frames[rel_path]
+        raw = self._read_bytes(rel_path)
+        if rel_path.endswith(".gz"):
+            with gzip.open(io.BytesIO(raw), "rt", encoding="utf-8") as handle:
+                frame = pd.read_csv(handle)
+        else:
+            frame = pd.read_csv(io.BytesIO(raw))
+        self._frames[rel_path] = frame
+        return frame
+
+    def _read_text(self, rel_path: str) -> str:
+        return self._read_bytes(rel_path).decode("utf-8")
+
+    def _read_bytes(self, rel_path: str) -> bytes:
+        if self.local_root is not None:
+            local = self.local_root / rel_path
+            if local.is_file():
+                return local.read_bytes()
+        return self._download(rel_path)
+
+    def _download(self, rel_path: str) -> bytes:
+        if rel_path not in self._byte_cache:
+            from huggingface_hub import hf_hub_download
+
+            path = hf_hub_download(
+                repo_id=self.repo_id,
+                filename=rel_path,
+                repo_type="dataset",
+                revision=self.revision,
+                cache_dir=self.cache_dir,
+            )
+            self._byte_cache[rel_path] = Path(path).read_bytes()
+        return self._byte_cache[rel_path]
+
+
+def _frame_payload(frame: pd.DataFrame, limit: int = 500) -> dict[str, Any]:
+    if frame.empty:
+        return {"count": 0, "rows": []}
+    trimmed = frame.head(limit)
+    return {
+        "count": int(len(frame)),
+        "truncated": len(frame) > limit,
+        "rows": json.loads(trimmed.to_json(orient="records")),
+    }

requirements.txt

@@ -0,0 +1,7 @@
+fastapi>=0.110.0
+uvicorn[standard]>=0.27.0
+gradio>=4.44.0
+huggingface_hub>=0.20.0
+pandas>=2.1.0
+statsmodels>=0.14.0
+pydantic>=2.6.0