"""MIO-style four-modality tokenizer allocation + streaming decode latency calc.
Stdlib. Prints the vocab layout and a step-by-step latency trace for a
spoken-dialogue request where MIO consumes speech, generates speech.
"""
from __future__ import annotations
from dataclasses import dataclass
@dataclass
class VocabSlot:
name: str
start: int
size: int
@property
def end(self) -> int:
return self.start + self.size
def build_vocab() -> list[VocabSlot]:
slots = []
cursor = 0
plan = [
("text BPE", 32000),
("image SEED", 8192),
("speech 4x1024", 4096),
("music", 8192),
("", 1),
("", 1),
("", 1),
("", 1),
("", 1),
("", 1),
]
for name, size in plan:
slots.append(VocabSlot(name=name, start=cursor, size=size))
cursor += size
return slots
def print_vocab(slots: list[VocabSlot]) -> None:
print("\nSHARED VOCABULARY LAYOUT")
print("-" * 60)
print(f" {'slot':<18}{'start':>8}{'end':>8}{'size':>8}")
for s in slots:
print(f" {s.name:<18}{s.start:>8}{s.end:>8}{s.size:>8}")
total = slots[-1].end
print(f" {'TOTAL':<18}{total:>8}{'(vocab size)':>16}")
def route_inputs(inputs: list[dict]) -> list[dict]:
"""Classify each input and assign a tokenizer path."""
routed = []
for inp in inputs:
kind = inp["kind"]
if kind == "text":
path = "BPE"
elif kind == "image":
path = "SEED-Tokenizer"
elif kind in ("speech", "voice"):
path = "SpeechTokenizer residual-VQ"
elif kind == "music":
path = "Encodec"
else:
path = "UNKNOWN"
routed.append({**inp, "path": path})
return routed
@dataclass
class LatencyTrace:
label: str
ms: float
def streaming_decode_latency(
prompt_audio_seconds: float = 2.0,
model_size_b: int = 8,
) -> list[LatencyTrace]:
trace = []
trace.append(LatencyTrace("mic audio -> speech tokens",
prompt_audio_seconds * 20))
trace.append(LatencyTrace("prefill prompt tokens",
80 * (model_size_b / 8.0)))
trace.append(LatencyTrace("first output token",
40 * (model_size_b / 8.0)))
trace.append(LatencyTrace("residual-VQ layers 1..7",
30))
trace.append(LatencyTrace("speech decoder (Encodec-like)",
80))
return trace
def print_trace(trace: list[LatencyTrace]) -> None:
print("\nSTREAMING DECODE LATENCY (time-to-first-audio-byte)")
print("-" * 60)
total = 0.0
for t in trace:
total += t.ms
print(f" {t.label:<38} +{t.ms:>5.0f} ms (cumul {total:>6.0f})")
print("-" * 60)
print(f" total TTFAB: {total:.0f} ms")
if total < 500:
print(f" -> conversational feel (GPT-4o-class)")
elif total < 800:
print(f" -> acceptable (first-gen open any-to-any)")
else:
print(f" -> sluggish, consider smaller model or parallel decode")
def demo_chain_of_visual_thought() -> None:
print("\nCHAIN-OF-VISUAL-THOUGHT (MIO)")
print("-" * 60)
prompt = "Is the cat climbing the tree in this photo?"
steps = [
"user text -> vision tokens",
"model sketches intermediate image ... ",
"model emits text analysis of sketch",
"model concludes with yes/no + justification",
]
print(f" prompt: {prompt}")
for i, s in enumerate(steps, 1):
print(f" step {i}: {s}")
print(" wins on spatial-reasoning benchmarks; hurts latency.")
def main() -> None:
print("=" * 60)
print("MIO ANY-TO-ANY STREAMING (Phase 12, Lesson 16)")
print("=" * 60)
vocab = build_vocab()
print_vocab(vocab)
print("\nROUTER: four inputs -> four tokenizers")
print("-" * 60)
inputs = [
{"kind": "text", "payload": "Hello"},
{"kind": "image", "payload": "cat.png"},
{"kind": "voice", "payload": "user.wav"},
{"kind": "music", "payload": "loop.mp3"},
]
for r in route_inputs(inputs):
print(f" {r['kind']:<8} '{r['payload']}' -> {r['path']}")
trace = streaming_decode_latency(prompt_audio_seconds=2.0, model_size_b=8)
print_trace(trace)
demo_chain_of_visual_thought()
if __name__ == "__main__":
main()