(() => { 'use strict'; const root = document.getElementById('moe-architecture'); if (!root) return; root.innerHTML = `
Inside the model / interactive architecture

A big model. A small working set.

Follow the weights from SSD to execution. Select a layer or a component to inspect what lives there and which kernels run.

Qwen3.8 · M5 · 32 GB
48 layers · teal = Gated DeltaNet · violet = full attention
Token embedding → 48 × [ hyper-connection → token mixer → hyper-connection → MoE ] → norm → lm_head
01 / SSD persistent weights
02 / UNIFIED MEMORY CPU + GPU share 32 GB
HOST / CPU
SSD → pinned buffers
contiguous expert reads
Look ahead → L+1 / L+2
predict, read, install ahead

OS page cache also uses RAM. These are logical allocations, not separate RAM and VRAM chips.

DEVICE / MPS
Resident / cache hitDisk read / cache missKernel inspection

Component inspector

Read implementation ↗

Illustrative routing and slowed-down animation; not a live trace or a latency simulation. Cache size changes capacity, not a predicted hit rate. Weight sizes follow the published report; runtime allocations and OS memory vary. RFC · Project

`; const $ = selector => root.querySelector(selector); const $$ = selector => [...root.querySelectorAll(selector)]; const repo = 'https://github.com/tarekziade/moe-expert-streaming/blob/main/src/moe_expert_streaming/'; const upstream = 'https://github.com/huggingface/transformers/pull/50717'; const mlx = 'https://huggingface.co/kernels-community/mlx-quantization-metal-kernels'; let layer = 0, stage = 0, mode = 'decode', selected = 'experts', timer = null; const stageNames = ['Mix', 'Route', 'Read', 'Install', 'Compute', 'Continue']; const layers = $('.mv-layers'); for (let i = 0; i < 48; i++) { const button = document.createElement('button'); button.type = 'button'; button.className = `mv-layer${i % 4 === 3 ? ' full' : ''}`; button.setAttribute('aria-label', `Layer ${i + 1}: ${i % 4 === 3 ? 'full attention' : 'Gated DeltaNet'}`); button.title = button.getAttribute('aria-label'); button.addEventListener('click', () => { stop(); layer = i; stage = 0; selected = 'mixer'; render(); }); layers.append(button); } stageNames.forEach((name, i) => { const button = document.createElement('button'); button.type = 'button'; button.className = 'mv-stage'; button.textContent = `0${i + 1} ${name}`; button.addEventListener('click', () => { stop(); stage = i; render(); }); $('.mv-timeline').append(button); }); for (let i = 0; i < 512; i++) { const dot = document.createElement('i'); dot.className = 'mv-dot'; $('.mv-grid').append(dot); } for (let i = 0; i < 48; i++) $('.mv-cache-grid').append(document.createElement('i')); function details() { const prefill = mode === 'prefill', full = layer % 4 === 3; return { experts: ['Routed experts / SSD', 'All 24,576 experts stay on disk. Each selected expert brings its packed weights, scales and biases into the bounded pool; a hit reuses an existing slot.', '3-bit MLX affine · ~2.15 MB per expert', 'Only 10 of 512 experts are selected per token per layer. The dots show one layer; highlighted choices are illustrative.', upstream], ngram: ['N-gram embedding / SSD', 'The 51 GB FP8 table is memory-mapped. Layer 1 requests a few rows, which are dequantized and copied to the device. It does not use the expert LRU cache.', 'DiskEmbedding.forward · NumPy memmap → PyTorch', 'OS page-cache-backed row reads, FP8-to-float conversion, scale, then conversion to the model dtype.', repo + 'qwen38.py'], host: ['Routing & cache management / CPU', 'One routing copy to the host per streamed MoE layer identifies hits and misses. The shared expert can execute while the CPU manages the cache.', 'Host code · expert-to-slot table · LRU', 'Eight I/O workers read misses. Decode also runs the next two routers on current hidden states to guess future experts; these are predictions, not future routing results.', upstream], staging: ['Pinned staging / host allocation', 'Recycled pinned buffers hold expert reads before non-blocking copies install them into the preallocated MPS pool.', 'Contiguous read → pinned buffer → device slot', 'Host staging and MPS tensors share physical unified memory on this Mac. The logical copy still exists. The OS page cache is separate from the explicit device LRU.', upstream], cache: ['Expert slot pool / MPS', 'One global pool is shared by all 48 layers. It stores quantized weights, scales and biases. LRU eviction reuses slots; the persistent SSD copy remains.', 'Slot-indexed gather_qmm', '12 GiB provides about 5,991 slots (~24% of all experts). The diagram shows a sample of slots, not occupancy telemetry. More slots do not imply a specific hit rate.', upstream], router: ['Router / MPS → CPU', 'The BF16 router chooses the top 10 experts and their mixing weights for each token. Routing IDs cross to the host to resolve cache slots.', 'PyTorch MPS ops · top-k', 'The single small host synchronization per streamed MoE layer is a real decode cost. Prediction does not replace the actual router.', upstream], shared: ['Shared expert / resident on MPS', 'One shared expert per layer runs for every token, alongside the routed experts. Its dense linear weights remain resident in 8-bit.', 'quantized_matmul · activation · gate', prefill ? 'Prefill uses float32 inputs for quantized matrix calls to avoid the reported M5 multi-row precision issue.' : 'The shared expert is queued after the routing copy so its GPU work overlaps CPU cache bookkeeping.', mlx], compute: ['Routed experts / MPS compute', prefill ? 'Prompt rows are sorted by expert. Groups of 64 experts bound the transient bank, since a long prompt touches nearly every expert.' : 'The selected experts run directly from cache slots, then their outputs are combined using the router weights.', prefill ? 'sorted gather_qmm · float32 · groups of 64 experts' : 'gather_qmm (unsorted) · gate/up → SiLU × up → down', prefill ? 'Float32 avoids the reported M5 BF16/FP16 multi-row issue. “64” is the expert group size, not the number of experts selected by one token.' : 'MLX Metal kernels read packed 3-bit weights from the pool. Single-row decode uses the quantized decode path.', mlx], mixer: [full ? 'Full attention / 12 of 48 layers' : 'Gated DeltaNet / 36 of 48 layers', full ? 'Every fourth layer uses sparse full attention. The indexer retains at most 2,048 prior tokens per query; selection is skipped when every key fits.' : 'Linear attention maintains recurrent state. Its large projections remain resident in 8-bit; recurrence state is updated on device.', full ? 'quantized_matmul + PyTorch indexer + MPS scaled_dot_product_attention' : prefill ? 'quantized_matmul (float32) + PyTorch chunked gated delta rule' : 'merged in_proj → fused conv/gates → fused delta rule + gated RMSNorm → out_proj', full ? 'The sparse attention mask uses the MPS SDPA path; the flash kernels do not accept this mask.' : prefill ? 'On macOS, prefill convolution, gated RMSNorm and recurrence use the PyTorch paths. The Triton flash-linear-attention path is unavailable.' : 'About 34 operations become 4 dispatches: two quantized matrix calls and two custom Metal kernels. Delta-rule state is float32.', full ? repo + 'qwen38.py' : repo + 'kernels/gated_delta_net.py'], hyper: ['Hyper-connections / resident on MPS', 'Four residual streams wrap the token mixer and MoE block. The projections are 8-bit; stream mixing stays on device.', prefill ? 'PyTorch forward + quantized projections (float32)' : 'Metal RMSNorm · SiLU · stream mixing + quantized_matmul', prefill ? 'The custom step kernel is for single-token decode. Multi-token prefill retains the original model forward.' : 'The 10,240 → 320 and → 4 projections share one matrix call. Custom Metal kernels implement normalization, activation and stream mixing.', repo + 'kernels/hyper_connection.py'], resident: ['Resident weights & runtime state', '~5.3 GB of 8-bit dense weights plus ~1.3 GB of BF16 embeddings, norms, routers, convolution weights and small gates, as reported by the Space.', 'quantized_matmul + PyTorch MPS ops', 'KV caches, recurrent states and activations also use memory and depend on workload. Published device memory after load is 17.5 GiB with the 12 GiB pool. MTP and vision are not loaded.', repo + 'metal_linear.py'] }; } function render() { const prefill = mode === 'prefill', budget = Number($('#mv-budget').value); $('#mv-layer-label').textContent = `LAYER ${String(layer + 1).padStart(2, '0')} / 48 · ${layer % 4 === 3 ? 'FULL ATTENTION' : 'GATED DELTANET'}`; $$('.mv-layer').forEach((b, i) => b.setAttribute('aria-pressed', String(i === layer))); $$('.mv-stage').forEach((b, i) => b.setAttribute('aria-pressed', String(i === stage))); $('#mv-decode').setAttribute('aria-pressed', String(!prefill)); $('#mv-prefill').setAttribute('aria-pressed', String(prefill)); $('#mv-mixer-title').textContent = layer % 4 === 3 ? 'Full attention · sparse indexer' : 'Gated DeltaNet · linear attention'; $('#mv-cache-title').textContent = `${budget} GiB · shared LRU pool`; $('#mv-cache-caption').textContent = `≈${Math.floor(5991 * budget / 12).toLocaleString('en-US')} slots / 24,576 experts · sample below`; $('#mv-compute-caption').textContent = prefill ? 'sorted gather_qmm · float32 · 64 experts / group' : 'gather_qmm · 10 selected experts / token'; const picks = Array.from({length:10}, (_, i) => (layer * 37 + i * 47 + 11) % 512); $$('.mv-dot').forEach((dot, i) => { const chosen = prefill ? i < 64 : picks.includes(i); dot.className = 'mv-dot' + (stage >= 1 && chosen ? ((prefill || picks.indexOf(i) >= 8) && stage < 3 ? ' miss' : ' chosen') : ''); }); $('#mv-expert-caption').textContent = prefill ? '512 experts in this layer · one 64-expert group highlighted' : '512 experts in this layer · 10 illustrative routes highlighted'; $$('.mv-cache-grid i').forEach((dot,i) => { dot.className = stage >= 1 && i < 8 ? 'hit' : stage >= 3 && i < 10 ? 'new' : ''; }); const active = [['mixer','hyper'], ['router','host','shared'], ['host','staging'], ['staging','cache'], ['cache','compute','shared'], ['hyper','resident']][stage]; $$('[data-part]').forEach(node => { node.classList.toggle('active', active.includes(node.dataset.part)); node.classList.toggle('reading', stage === 2 && node.dataset.part === 'experts'); node.classList.toggle('selected', node.dataset.part === selected); node.setAttribute('aria-pressed', String(node.dataset.part === selected)); }); $('#mv-read').classList.toggle('running', stage === 2 || stage === 3); $('#mv-prefetch').classList.toggle('running', !prefill && stage >= 2 && stage <= 4 && layer < 47); $('#mv-prefetch').textContent = prefill ? 'Prefill → expert groups\nno decode look-ahead shown' : layer === 47 ? 'Final layer · no future layer to prefetch' : `Look ahead → ${layer === 46 ? 'L48' : `L${layer + 2} / L${layer + 3}`} · predict & read`; const narratives = prefill ? [ ['Process a chunk of prompt tokens', 'Token mixing runs the prefill path. Quantized matrix calls use float32; Gated DeltaNet recurrence uses PyTorch on this Mac.'], ['Route each token, then group by expert', 'Each token still selects 10 experts. Across a long prompt, the union touches nearly all 512 experts in each layer.'], ['Read the next expert group', 'Cache hits can be reused; missing weights are read from SSD. The highlighted group is illustrative, not recorded routing.'], ['Bound the transient expert bank', 'Expert groups are assembled for sorted execution, up to 64 experts at a time. This avoids a full-layer transient allocation.'], ['Execute sorted gather_qmm in float32', 'Rows are sorted by expert, evaluated, weighted, and accumulated back into token order. Shared-expert output is also included.'], ['Repeat groups, then continue the model', 'All routed groups complete before the layer output proceeds. The next layer repeats this process; the last layer feeds the output head.'] ] : [ ['Mix the current token', `Layer ${layer + 1} runs ${layer % 4 === 3 ? 'sparse full attention' : 'Gated DeltaNet'} inside its hyper-connection. The dense weights stay on device.${layer === 0 ? ' Layer 1 also reads the requested n-gram rows from SSD.' : ''}`], ['Route to 10 experts, resolve their slots', 'Illustration: 8 hits and 2 misses. Routing IDs go to the CPU once; the resident shared expert overlaps cache bookkeeping.'], ['Fetch only missing experts', 'Eight I/O workers read contiguous expert records into recycled pinned buffers. Future-layer guesses can already be reading in the background.'], ['Install weights into reusable slots', 'Non-blocking copies populate the MPS pool. LRU entries can be evicted, while the complete expert bank remains on SSD.'], ['Execute directly from the slot pool', 'Unsorted gather_qmm applies the selected 3-bit experts; router weights combine their outputs. The shared expert contributes too.'], [layer === 47 ? 'Finish with norm and lm_head' : 'Carry the token to the next layer', 'Hyper-connections mix the streams. Early MPS commits let queued GPU work start while the CPU continues; the cache persists across layers and tokens.'] ]; $('#mv-stage-title').textContent = narratives[stage][0]; $('#mv-stage-description').textContent = narratives[stage][1]; const info = details()[selected]; $('#mv-detail-title').textContent = info[0]; $('#mv-detail-description').textContent = info[1]; $('#mv-kernel-label').textContent = `${mode} / implementation`; $('#mv-kernel').textContent = info[2]; $('#mv-kernel-description').textContent = info[3]; $('#mv-source').href = info[4]; } function stop() { clearInterval(timer); timer = null; $('#mv-play').textContent = '▶ Play flow'; } function advance() { stage = (stage + 1) % 6; if (!stage) layer = (layer + 1) % 48; render(); } $('#mv-play').addEventListener('click', () => { if (timer) stop(); else { timer = setInterval(advance, 2400); $('#mv-play').textContent = 'Ⅱ Pause'; } }); $('#mv-next').addEventListener('click', () => { stop(); advance(); }); for (const name of ['decode','prefill']) $(`#mv-${name}`).addEventListener('click', () => { stop(); mode = name; stage = 0; render(); }); $('#mv-budget').addEventListener('change', render); $$('[data-part]').forEach(node => node.addEventListener('click', () => { selected = node.dataset.part; render(); })); document.addEventListener('visibilitychange', () => { if (document.hidden) stop(); }); render(); })();