j-chim Claude Fable 5 commited on
Commit
68d591a
·
1 Parent(s): 0ab5b2a

Collections groundwork: uk-aisi logo asset + additive snapshot plumbing

Browse files

- public/org-logos/uk-aisi.svg — served at the path the registry's
canonical_orgs.logo_url points to (evaluator letterhead).
- Optional collection fields on ModelResultForBenchmark (not yet
populated by any adapter — display integration is spec'd separately).
- collection_trajectories registered as an optional snapshot table
(best-effort load; absent on older snapshots).
- Integrity suite: collections.json presence/coverage checks,
self-skipping on pre-collections snapshots.

No behavior change on any page.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

lib/duckdb.ts CHANGED
@@ -35,6 +35,10 @@ const VIEW_FILES = {
35
  // is absent.
36
  const OPTIONAL_VIEW_FILES = {
37
  merged_evals_view: "merged_evals_view.parquet",
 
 
 
 
38
  } as const
39
 
40
  export async function getConnection(): Promise<DuckDBConnection> {
 
35
  // is absent.
36
  const OPTIONAL_VIEW_FILES = {
37
  merged_evals_view: "merged_evals_view.parquet",
38
+ // Collections (notes/collection-dashboard-spec.md): per-attempt
39
+ // trajectories for protocol-varied collections; only present on
40
+ // snapshots whose backend shipped a vendored collection extract.
41
+ collection_trajectories: "collection_trajectories.parquet",
42
  } as const
43
 
44
  export async function getConnection(): Promise<DuckDBConnection> {
lib/eval-processing.ts CHANGED
@@ -47,6 +47,16 @@ export interface ModelResultForBenchmark {
47
  /** Merged-page rows only (set by lib/merged-adapter): the observation's
48
  * source composite slug, used for the ?source= row pre-highlight. */
49
  merged_source_slug?: string
 
 
 
 
 
 
 
 
 
 
50
  aggregate_components?: Array<{
51
  evaluation_id: string
52
  composite_benchmark_key: string
 
47
  /** Merged-page rows only (set by lib/merged-adapter): the observation's
48
  * source composite slug, used for the ?source= row pre-highlight. */
49
  merged_source_slug?: string
50
+ /** Collections (see the backend's collections spec, warehouse-outputs section): submission-channel id of
51
+ * the row's representative fact row — key into the snapshot's
52
+ * `collections.json` sidecar. */
53
+ collection_id?: string
54
+ /** Protocol point for protocol-varied collections: canonical sorted-key
55
+ * JSON typed by the collection's declared `protocol_axes`; absent/null
56
+ * for ordinary rows. The view ships one row per protocol point; rows
57
+ * whose reserved `feedback` key is `answer_feedback` are
58
+ * shown-but-not-ranked (backend already emits NULL position for them). */
59
+ protocol_condition?: string | null
60
  aggregate_components?: Array<{
61
  evaluation_id: string
62
  composite_benchmark_key: string
public/org-logos/uk-aisi.svg ADDED
tests/redirect-integrity.test.ts CHANGED
@@ -91,3 +91,58 @@ describe.skipIf(!shouldRun)("redirect-map + fallback integrity (vs SNAPSHOT_URL)
91
  expect(nullRaw, `${nullRaw}/${rowCount} rows have NULL raw_model_ids`).toBe(0)
92
  })
93
  })
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  expect(nullRaw, `${nullRaw}/${rowCount} rows have NULL raw_model_ids`).toBe(0)
92
  })
93
  })
94
+
95
+ // ---------------------------------------------------------------------------
96
+ // collections.json integrity (backend collections spec, registry-sidecar section)
97
+ // ---------------------------------------------------------------------------
98
+ //
99
+ // Snapshots that carry the collections columns must also ship a parseable
100
+ // `collections.json` covering every collection_id the view references.
101
+ // Pre-collections snapshots (no `collection_id` column) self-skip so this
102
+ // suite stays usable for older pins.
103
+
104
+ describe.skipIf(!shouldRun)("collections sidecar integrity (vs SNAPSHOT_URL)", () => {
105
+ let hasCollections = false
106
+ let viewIds: string[] = []
107
+ let registry: Record<string, { display_name?: string; curated?: boolean }> | null = null
108
+
109
+ beforeAll(async () => {
110
+ const con = await DuckDBConnection.create()
111
+ await con.run("INSTALL httpfs; LOAD httpfs;")
112
+ const cols = (await con.runAndReadAll(
113
+ `SELECT column_name FROM (DESCRIBE SELECT * FROM read_parquet('${SNAPSHOT}/eval_results_view.parquet'))`,
114
+ )).getRowObjects().map((r) => String(r.column_name))
115
+ hasCollections = cols.includes("collection_id") && cols.includes("protocol_condition")
116
+ if (!hasCollections) return
117
+ viewIds = (await con.runAndReadAll(
118
+ `SELECT DISTINCT collection_id FROM read_parquet('${SNAPSHOT}/eval_results_view.parquet')
119
+ WHERE collection_id IS NOT NULL`,
120
+ )).getRowObjects().map((r) => String(r.collection_id))
121
+ if (SNAPSHOT.startsWith("http")) {
122
+ try {
123
+ const res = await fetch(`${SNAPSHOT}/collections.json`)
124
+ registry = res.ok ? await res.json() : null
125
+ } catch {
126
+ registry = null
127
+ }
128
+ } else {
129
+ const path = SNAPSHOT.replace(/^file:\/\//, "")
130
+ try {
131
+ registry = JSON.parse(readFileSync(`${path}/collections.json`, "utf8"))
132
+ } catch {
133
+ registry = null
134
+ }
135
+ }
136
+ }, 120_000)
137
+
138
+ it("collections.json is present and parseable when the view carries collection columns", () => {
139
+ if (!hasCollections) return
140
+ expect(registry, "collections.json missing/unparseable").toBeTruthy()
141
+ })
142
+
143
+ it("every view collection_id has a registry entry with a display name", () => {
144
+ if (!hasCollections || !registry) return
145
+ const missing = viewIds.filter((id) => !registry![id]?.display_name)
146
+ expect(missing, `unregistered collection ids: ${missing.slice(0, 8)}`).toEqual([])
147
+ })
148
+ })