feat(metrics): instrument per-stage and pipeline_total latency; count pipeline_runs_total (ok/error/ambiguous)

Makefile

@@ -108,3 +108,24 @@ clean: ## Remove Python caches
 .PHONY: clean-all
 clean-all: clean ## Remove build artifacts and coverage
 	rm -rf dist build .coverage *.egg-info
+
+# ---------- Metrics ----------
+.PHONY: prom-up prom-check smoke
+
+prom-up:
+	docker compose -f docker-compose.prom.yml up -d
+
+prom-check:
+	@if command -v promtool >/dev/null 2>&1; then \
+		echo "🔍 Running promtool locally..."; \
+		promtool check rules prometheus/rules.yml && promtool check config prometheus/prometheus.yml; \
+	else \
+		echo "⚠️ promtool not found, running via Docker..."; \
+		docker run --rm -v $$(pwd)/prometheus:/etc/prometheus prom/prometheus \
+			promtool check rules /etc/prometheus/rules.yml && \
+		docker run --rm -v $$(pwd)/prometheus:/etc/prometheus prom/prometheus \
+			promtool check config /etc/prometheus/prometheus.yml; \
+	fi
+
+smoke:
+	./scripts/smoke_metrics.sh

docker-compose.prom.yml

@@ -0,0 +1,13 @@
+version: "3.8"
+services:
+  prometheus:
+    image: prom/prometheus:v2.55.0
+    container_name: nl2sql-prom
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus/rules.yml:/etc/prometheus/rules.yml:ro
+    ports:
+      - "9090:9090"
+    restart: unless-stopped

prometheus/prometheus.yml

@@ -0,0 +1,14 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+# Load recording and alerting rules from this file
+rule_files:
+  - rules.yml   # relative to this directory (prometheus/)
+
+scrape_configs:
+  - job_name: "nl2sql-app"
+    static_configs:
+      - targets: ["host.docker.internal:8000"]  # use this if Prometheus runs inside Docker
+        labels:
+          service: "nl2sql"

prometheus/rules.yml

@@ -0,0 +1,49 @@
+groups:
+- name: nl2sql_latency
+  rules:
+  # p95 latency per stage (5-minute window)
+  - record: nl2sql:stage_p95_ms
+    expr: |
+      histogram_quantile(
+        0.95,
+        sum(rate(stage_duration_ms_bucket[5m])) by (le, stage)
+      )
+
+  # pipeline success ratio (5-minute rolling window)
+  - record: nl2sql:pipeline_success_ratio
+    expr: |
+      sum(rate(pipeline_runs_total{status="ok"}[5m]))
+      /
+      sum(rate(pipeline_runs_total[5m]))
+
+- name: nl2sql_alerts
+  rules:
+  # Alert: success ratio below 90% for 10 minutes
+  - alert: PipelineLowSuccessRatio
+    expr: nl2sql:pipeline_success_ratio < 0.9
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Pipeline success ratio dropped"
+      description: "Success ratio < 90% over the past 10 minutes"
+
+  # Alert: high generator p95 latency (>1.5s for 5 minutes)
+  - alert: GeneratorLatencyHigh
+    expr: nl2sql:stage_p95_ms{stage="generator"} > 1500
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Generator p95 latency high"
+      description: "Generator p95 > 1.5s for 5 minutes"
+
+  # Alert: unusual spike in Safety blocks
+  - alert: SafetyBlocksSpike
+    expr: rate(safety_blocks_total[5m]) > 0.5
+    for: 5m
+    labels:
+      severity: info
+    annotations:
+      summary: "Unusual Safety block rate"
+      description: "Safety blocks rate > 0.5 per minute (check inputs or safety rules)"

scripts/smoke_metrics.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BASE=${BASE:-http://localhost:8000}
+API="$BASE/api/v1"
+
+# Send a few successful queries to populate basic metrics
+for q in \
+  "List all artists" \
+  "Top 5 albums by sales" \
+  "Count customers"
+do
+  curl -s -X POST "$API/nl2sql" \
+    -H 'Content-Type: application/json' \
+    -d "{\"query\":\"$q\"}" >/dev/null || true
+done
+
+# Send queries that trigger safety and verifier checks
+curl -s -X POST "$API/nl2sql" \
+  -H 'Content-Type: application/json' \
+  -d '{"query":"DELETE FROM users;"}' >/dev/null || true
+
+curl -s -X POST "$API/nl2sql" \
+  -H 'Content-Type: application/json' \
+  -d '{"query":"SELECT COUNT(*), country FROM customers;"}' >/dev/null || true
+
+# Print a snapshot of key Prometheus metrics
+echo -e "\n--- Metrics snapshot ---"
+curl -s "$BASE/metrics" | grep -E \
+  'stage_duration_ms_(sum|count|bucket)|pipeline_runs_total|safety_(checks|blocks)_total|verifier_(checks|failures)_total' \
+  || true