"""Original Token Lab byte-level causal LM. Python 3.12 / torch 2.5.1 CPU.
Run: python tiny_lm.py --steps 300 --output tiny.pt
No pretrained weights, downloads, model API or third-party dataset.
"""
import argparse
import math
from pathlib import Path
import torch
from torch import nn
from torch.nn import functional as F

CFG = dict(vocab=256, context=32, width=64, heads=4, layers=2)

class Attention(nn.Module):
    def __init__(self, width, heads, context):
        super().__init__()
        assert width % heads == 0
        self.heads, self.dim = heads, width // heads
        self.qkv = nn.Linear(width, 3 * width)
        self.out = nn.Linear(width, width)
        self.register_buffer('mask', torch.ones(context, context, dtype=torch.bool).tril())

    def forward(self, x):
        b, t, c = x.shape
        q, k, v = self.qkv(x).chunk(3, dim=-1)
        q, k, v = [a.view(b, t, self.heads, self.dim).transpose(1, 2) for a in (q, k, v)]
        scores = q @ k.transpose(-2, -1) / math.sqrt(self.dim)
        scores = scores.masked_fill(~self.mask[:t, :t], float('-inf'))
        mixed = scores.softmax(dim=-1) @ v
        return self.out(mixed.transpose(1, 2).contiguous().view(b, t, c))

class Block(nn.Module):
    def __init__(self, c):
        super().__init__()
        d=c['width']
        self.n1, self.n2 = nn.LayerNorm(d), nn.LayerNorm(d)
        self.attn = Attention(d, c['heads'], c['context'])
        self.ff = nn.Sequential(nn.Linear(d, 4*d), nn.GELU(), nn.Linear(4*d, d))

    def forward(self, x):
        x = x + self.attn(self.n1(x))
        return x + self.ff(self.n2(x))

class TinyLM(nn.Module):
    def __init__(self, cfg=CFG):
        super().__init__()
        self.cfg = cfg
        self.tokens = nn.Embedding(cfg['vocab'], cfg['width'])
        self.positions = nn.Embedding(cfg['context'], cfg['width'])
        self.blocks = nn.Sequential(*[Block(cfg) for _ in range(cfg['layers'])])
        self.norm = nn.LayerNorm(cfg['width'])
        self.head = nn.Linear(cfg['width'], cfg['vocab'], bias=False)
        self.apply(self.init)

    @staticmethod
    def init(m):
        if isinstance(m, (nn.Linear, nn.Embedding)):
            nn.init.normal_(m.weight, std=.02)
            if getattr(m, 'bias', None) is not None: nn.init.zeros_(m.bias)

    def forward(self, ids):
        if ids.ndim != 2 or not 0 < ids.shape[1] <= self.cfg['context']:
            raise ValueError('Expected [batch, time], 1 <= time <= context')
        x = self.tokens(ids) + self.positions(torch.arange(ids.shape[1], device=ids.device))
        return self.head(self.norm(self.blocks(x)))

    @torch.no_grad()
    def generate(self, prompt, count=80, temperature=.8, top_k=20):
        if not prompt or temperature <= 0 or not 1 <= top_k <= self.cfg['vocab']:
            raise ValueError('Nonempty prompt, positive temperature, valid top_k required')
        self.eval()
        ids = torch.tensor([list(prompt.encode('utf-8'))], dtype=torch.long)
        for _ in range(count):
            logits = self(ids[:, -self.cfg['context']:])[:, -1, :] / temperature
            threshold = logits.topk(top_k).values[:, -1:]
            logits = logits.masked_fill(logits < threshold, float('-inf'))
            ids = torch.cat((ids, torch.multinomial(logits.softmax(-1), 1)), dim=1)
        return bytes(ids[0].tolist()).decode('utf-8', errors='replace')

def corpus():
    # Original synthetic sentences. Split document IDs BEFORE window extraction.
    docs = [f'node {i:03d}: status is {"ready" if i % 2 == 0 else "busy"}.\n' for i in range(120)]
    return ''.join(docs[:100]).encode(), ''.join(docs[100:]).encode()

def batch(raw, generator, size=16):
    data=torch.tensor(list(raw), dtype=torch.long)
    t=CFG['context']
    starts=torch.randint(len(data)-t, (size,), generator=generator)
    return torch.stack([data[i:i+t] for i in starts]), torch.stack([data[i+1:i+t+1] for i in starts])

def loss(model, x, y):
    logits=model(x)
    return F.cross_entropy(logits.reshape(-1, CFG['vocab']), y.reshape(-1))

@torch.no_grad()
def heldout_loss(model, raw):
    model.eval()
    data=torch.tensor(list(raw),dtype=torch.long)
    total, count=0.,0
    for i in range(0,len(data)-1,CFG['context']):
        t=min(CFG['context'],len(data)-1-i)
        x,y=data[i:i+t][None,:],data[i+1:i+t+1][None,:]
        total += loss(model,x,y).item()*t
        count += t
    return total/count

def checks(model):
    model.eval()
    x=torch.arange(12)[None,:]
    changed=x.clone();changed[:,7:]=200
    with torch.no_grad():
        a,b=model(x),model(changed)
    assert a.shape == (1,12,256)
    assert torch.allclose(a[:,:7],b[:,:7],atol=1e-6), 'future leak'
    try:model(torch.zeros(1,33,dtype=torch.long))
    except ValueError:pass
    else:raise AssertionError('context overflow accepted')
    # One coordinate of the readout gradient, central finite difference.
    target=torch.arange(1,13)[None,:]
    model.zero_grad(set_to_none=True)
    loss(model,x,target).backward()
    analytical=model.head.weight.grad[1,0].item()
    original=model.head.weight[1,0].item();eps=.01
    with torch.no_grad():
        model.head.weight[1,0]=original+eps;plus=loss(model,x,target).item()
        model.head.weight[1,0]=original-eps;minus=loss(model,x,target).item()
        model.head.weight[1,0]=original
    assert abs(analytical-(plus-minus)/(2*eps))<.002, 'gradient mismatch'

def main():
    p=argparse.ArgumentParser();p.add_argument('--steps',type=int,default=300)
    p.add_argument('--output',default='tiny.pt');p.add_argument('--text',help='Optional owned UTF-8 corpus; chronological 90/10 split')
    args=p.parse_args()
    if args.steps<1:raise ValueError('steps must be positive')
    torch.set_num_threads(1);torch.manual_seed(42);torch.use_deterministic_algorithms(True)
    generator=torch.Generator().manual_seed(123)
    train,val=corpus()
    if args.text:
        raw=Path(args.text).read_text(encoding='utf-8').encode();cut=int(.9*len(raw));train,val=raw[:cut],raw[cut:]
    if min(len(train),len(val))<=CFG['context']:raise ValueError('each split must exceed context')
    model=TinyLM();checks(model)
    opt=torch.optim.AdamW(model.parameters(),lr=.002,weight_decay=.01)
    before=heldout_loss(model,val)
    for step in range(args.steps):
        model.train();x,y=batch(train,generator)
        lr=.002*min((step+1)/20,1)*(.1+.9*.5*(1+math.cos(math.pi*step/args.steps)))
        for group in opt.param_groups:group['lr']=lr
        opt.zero_grad(set_to_none=True);value=loss(model,x,y)
        if not torch.isfinite(value):raise RuntimeError('nonfinite loss')
        value.backward();nn.utils.clip_grad_norm_(model.parameters(),1.0);opt.step()
    after=heldout_loss(model,val)
    torch.save({'config':CFG,'model':model.state_dict(),'optimizer':opt.state_dict(),
                'step':args.steps,'rng':torch.get_rng_state(),'batch_rng':generator.get_state()},args.output)
    saved=torch.load(args.output,weights_only=True);restored=TinyLM(saved['config']);restored.load_state_dict(saved['model']);restored.eval();model.eval()
    probe=torch.tensor([[1,2,3]])
    assert torch.equal(model(probe),restored(probe)), 'checkpoint mismatch'
    checks(model)
    print(f'parameters={sum(p.numel() for p in model.parameters())}')
    print(f'validation_nats_per_byte: {before:.6f} -> {after:.6f}')
    print(f'validation_perplexity_per_byte={math.exp(after):.6f}')
    print('checks: shapes, causal mask, context bound, gradient, checkpoint passed')
    if not args.text and args.steps>=300:assert after<before, 'synthetic learning failed'
    torch.manual_seed(42);print('sample='+repr(model.generate('node 10')))

if __name__=='__main__':main()
