Spaces:
Running
Running
Collections groundwork: uk-aisi logo asset + additive snapshot plumbing
Browse files- public/org-logos/uk-aisi.svg — served at the path the registry's
canonical_orgs.logo_url points to (evaluator letterhead).
- Optional collection fields on ModelResultForBenchmark (not yet
populated by any adapter — display integration is spec'd separately).
- collection_trajectories registered as an optional snapshot table
(best-effort load; absent on older snapshots).
- Integrity suite: collections.json presence/coverage checks,
self-skipping on pre-collections snapshots.
No behavior change on any page.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
- lib/duckdb.ts +4 -0
- lib/eval-processing.ts +10 -0
- public/org-logos/uk-aisi.svg +33 -0
- tests/redirect-integrity.test.ts +55 -0
lib/duckdb.ts
CHANGED
|
@@ -35,6 +35,10 @@ const VIEW_FILES = {
|
|
| 35 |
// is absent.
|
| 36 |
const OPTIONAL_VIEW_FILES = {
|
| 37 |
merged_evals_view: "merged_evals_view.parquet",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
} as const
|
| 39 |
|
| 40 |
export async function getConnection(): Promise<DuckDBConnection> {
|
|
|
|
| 35 |
// is absent.
|
| 36 |
const OPTIONAL_VIEW_FILES = {
|
| 37 |
merged_evals_view: "merged_evals_view.parquet",
|
| 38 |
+
// Collections (notes/collection-dashboard-spec.md): per-attempt
|
| 39 |
+
// trajectories for protocol-varied collections; only present on
|
| 40 |
+
// snapshots whose backend shipped a vendored collection extract.
|
| 41 |
+
collection_trajectories: "collection_trajectories.parquet",
|
| 42 |
} as const
|
| 43 |
|
| 44 |
export async function getConnection(): Promise<DuckDBConnection> {
|
lib/eval-processing.ts
CHANGED
|
@@ -47,6 +47,16 @@ export interface ModelResultForBenchmark {
|
|
| 47 |
/** Merged-page rows only (set by lib/merged-adapter): the observation's
|
| 48 |
* source composite slug, used for the ?source= row pre-highlight. */
|
| 49 |
merged_source_slug?: string
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
aggregate_components?: Array<{
|
| 51 |
evaluation_id: string
|
| 52 |
composite_benchmark_key: string
|
|
|
|
| 47 |
/** Merged-page rows only (set by lib/merged-adapter): the observation's
|
| 48 |
* source composite slug, used for the ?source= row pre-highlight. */
|
| 49 |
merged_source_slug?: string
|
| 50 |
+
/** Collections (see the backend's collections spec, warehouse-outputs section): submission-channel id of
|
| 51 |
+
* the row's representative fact row — key into the snapshot's
|
| 52 |
+
* `collections.json` sidecar. */
|
| 53 |
+
collection_id?: string
|
| 54 |
+
/** Protocol point for protocol-varied collections: canonical sorted-key
|
| 55 |
+
* JSON typed by the collection's declared `protocol_axes`; absent/null
|
| 56 |
+
* for ordinary rows. The view ships one row per protocol point; rows
|
| 57 |
+
* whose reserved `feedback` key is `answer_feedback` are
|
| 58 |
+
* shown-but-not-ranked (backend already emits NULL position for them). */
|
| 59 |
+
protocol_condition?: string | null
|
| 60 |
aggregate_components?: Array<{
|
| 61 |
evaluation_id: string
|
| 62 |
composite_benchmark_key: string
|
public/org-logos/uk-aisi.svg
ADDED
|
|
tests/redirect-integrity.test.ts
CHANGED
|
@@ -91,3 +91,58 @@ describe.skipIf(!shouldRun)("redirect-map + fallback integrity (vs SNAPSHOT_URL)
|
|
| 91 |
expect(nullRaw, `${nullRaw}/${rowCount} rows have NULL raw_model_ids`).toBe(0)
|
| 92 |
})
|
| 93 |
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
expect(nullRaw, `${nullRaw}/${rowCount} rows have NULL raw_model_ids`).toBe(0)
|
| 92 |
})
|
| 93 |
})
|
| 94 |
+
|
| 95 |
+
// ---------------------------------------------------------------------------
|
| 96 |
+
// collections.json integrity (backend collections spec, registry-sidecar section)
|
| 97 |
+
// ---------------------------------------------------------------------------
|
| 98 |
+
//
|
| 99 |
+
// Snapshots that carry the collections columns must also ship a parseable
|
| 100 |
+
// `collections.json` covering every collection_id the view references.
|
| 101 |
+
// Pre-collections snapshots (no `collection_id` column) self-skip so this
|
| 102 |
+
// suite stays usable for older pins.
|
| 103 |
+
|
| 104 |
+
describe.skipIf(!shouldRun)("collections sidecar integrity (vs SNAPSHOT_URL)", () => {
|
| 105 |
+
let hasCollections = false
|
| 106 |
+
let viewIds: string[] = []
|
| 107 |
+
let registry: Record<string, { display_name?: string; curated?: boolean }> | null = null
|
| 108 |
+
|
| 109 |
+
beforeAll(async () => {
|
| 110 |
+
const con = await DuckDBConnection.create()
|
| 111 |
+
await con.run("INSTALL httpfs; LOAD httpfs;")
|
| 112 |
+
const cols = (await con.runAndReadAll(
|
| 113 |
+
`SELECT column_name FROM (DESCRIBE SELECT * FROM read_parquet('${SNAPSHOT}/eval_results_view.parquet'))`,
|
| 114 |
+
)).getRowObjects().map((r) => String(r.column_name))
|
| 115 |
+
hasCollections = cols.includes("collection_id") && cols.includes("protocol_condition")
|
| 116 |
+
if (!hasCollections) return
|
| 117 |
+
viewIds = (await con.runAndReadAll(
|
| 118 |
+
`SELECT DISTINCT collection_id FROM read_parquet('${SNAPSHOT}/eval_results_view.parquet')
|
| 119 |
+
WHERE collection_id IS NOT NULL`,
|
| 120 |
+
)).getRowObjects().map((r) => String(r.collection_id))
|
| 121 |
+
if (SNAPSHOT.startsWith("http")) {
|
| 122 |
+
try {
|
| 123 |
+
const res = await fetch(`${SNAPSHOT}/collections.json`)
|
| 124 |
+
registry = res.ok ? await res.json() : null
|
| 125 |
+
} catch {
|
| 126 |
+
registry = null
|
| 127 |
+
}
|
| 128 |
+
} else {
|
| 129 |
+
const path = SNAPSHOT.replace(/^file:\/\//, "")
|
| 130 |
+
try {
|
| 131 |
+
registry = JSON.parse(readFileSync(`${path}/collections.json`, "utf8"))
|
| 132 |
+
} catch {
|
| 133 |
+
registry = null
|
| 134 |
+
}
|
| 135 |
+
}
|
| 136 |
+
}, 120_000)
|
| 137 |
+
|
| 138 |
+
it("collections.json is present and parseable when the view carries collection columns", () => {
|
| 139 |
+
if (!hasCollections) return
|
| 140 |
+
expect(registry, "collections.json missing/unparseable").toBeTruthy()
|
| 141 |
+
})
|
| 142 |
+
|
| 143 |
+
it("every view collection_id has a registry entry with a display name", () => {
|
| 144 |
+
if (!hasCollections || !registry) return
|
| 145 |
+
const missing = viewIds.filter((id) => !registry![id]?.display_name)
|
| 146 |
+
expect(missing, `unregistered collection ids: ${missing.slice(0, 8)}`).toEqual([])
|
| 147 |
+
})
|
| 148 |
+
})
|