"""Transparent language-model lifecycle. Python 3.11 + numpy 2.3.5.
The model is a learned bigram softmax, not a Transformer or general assistant.
"""
import argparse, hashlib, json, os, tempfile
from pathlib import Path
import numpy as np

def digest(b): return hashlib.sha256(b).hexdigest()
def atomic(path, data):
    path=Path(path); path.parent.mkdir(parents=True,exist_ok=True)
    fd,tmp=tempfile.mkstemp(dir=path.parent)
    with os.fdopen(fd,'wb') as f: f.write(data); f.flush(); os.fsync(f.fileno())
    os.replace(tmp,path)
def prepare(source,out):
    records=[json.loads(x) for x in Path(source).read_text().splitlines() if x.strip()]
    seen=set(); groups={'train':[],'validation':[]}
    for r in records:
        if r.get('license')!='CC0' or not isinstance(r.get('text'),str): raise ValueError('unsupported rights/schema')
        text=' '.join(r['text'].split())
        if not text: raise ValueError('empty text')
        h=digest(text.encode())
        if h in seen: continue
        seen.add(h)
        # Split before windows; one source record never crosses splits.
        groups['validation' if int(h[:8],16)%5==0 else 'train'].append({'id':r['id'],'text':text,'sha256':h})
    if not all(groups.values()): raise ValueError('both splits required')
    payload=json.dumps(groups,sort_keys=True).encode()
    atomic(Path(out)/'dataset.json',payload)
    atomic(Path(out)/'data-manifest.json',json.dumps({'sha256':digest(payload),'source_sha256':digest(Path(source).read_bytes()),'split':'content-hash modulo 5','counts':{k:len(v) for k,v in groups.items()}},indent=2).encode())
    return groups

def pairs(records,vocab):
    lookup={c:i+2 for i,c in enumerate(vocab)}; xs=[];ys=[]
    for r in records:
        ids=[0]+[lookup[c] for c in r['text']]+[1]
        xs.extend(ids[:-1]);ys.extend(ids[1:])
    return np.array(xs),np.array(ys)
def loss(w,x,y):
    z=w[x]; z=z-z.max(axis=1,keepdims=True)
    return float(np.mean(np.log(np.exp(z).sum(axis=1))-z[np.arange(len(x)),y]))
def train(root,steps=400,resume=False):
    root=Path(root); raw=(root/'dataset.json').read_bytes();ds=json.loads(raw)
    # Vocabulary belongs to training data. Fixture is ASCII; production needs UNK policy.
    vocab=sorted(set(''.join(r['text'] for r in ds['train'])))
    if any(c not in vocab for r in ds['validation'] for c in r['text']): raise ValueError('validation has unknown characters')
    x,y=pairs(ds['train'],vocab); vx,vy=pairs(ds['validation'],vocab)
    rng=np.random.default_rng(42);w=np.zeros((len(vocab)+2,)*2);start=0
    cp=root/'checkpoint.npz'
    if resume and cp.exists():
        with np.load(cp,allow_pickle=False) as c:
            if str(c['data_hash'])!=digest(raw): raise ValueError('dataset changed during resume')
            w=c['weights'];start=int(c['step']);rng.bit_generator.state=json.loads(str(c['rng']))
    initial=loss(w,x,y)
    for step in range(start,steps):
        ix=rng.integers(0,len(x),size=128); xx=x[ix];yy=y[ix]
        z=w[xx];z-=z.max(axis=1,keepdims=True);p=np.exp(z);p/=p.sum(axis=1,keepdims=True)
        p[np.arange(len(xx)),yy]-=1;g=np.zeros_like(w);np.add.at(g,xx,p/len(xx));w-=3*g
        if (step+1)%50==0 or step+1==steps:
            import io
            buf=io.BytesIO();np.savez(buf,weights=w,step=step+1,rng=json.dumps(rng.bit_generator.state),data_hash=digest(raw));atomic(cp,buf.getvalue())
    model={'architecture':'bigram-softmax-v1','vocab':vocab,'weights':w.tolist(),'data_sha256':digest(raw),'steps':steps,'seed':42}
    modelraw=json.dumps(model,separators=(',',':')).encode();atomic(root/'model.json',modelraw)
    metrics={'initial_train_nll':initial,'train_nll':loss(w,x,y),'validation_nll':loss(w,vx,vy),'model_sha256':digest(modelraw),'steps':steps}
    atomic(root/'metrics.json',json.dumps(metrics,indent=2).encode());print(json.dumps(metrics));return metrics

def promote(root):
    root=Path(root);m=json.loads((root/'metrics.json').read_text()); raw=(root/'model.json').read_bytes()
    if digest(raw)!=m['model_sha256'] or not np.isfinite(m['validation_nll']) or m['validation_nll']>=4: raise ValueError('release gate failed')
    # This threshold only validates the declared teaching fixture, not language quality.
    atomic(root/'release.json',json.dumps({'model_sha256':digest(raw),'data_sha256':digest((root/'dataset.json').read_bytes()),'artifact':'model.json','gate':'fixture validation NLL < 4; not a safety evaluation'},indent=2).encode())

if __name__=='__main__':
    p=argparse.ArgumentParser();p.add_argument('action',choices=['prepare','train','promote']);p.add_argument('--root',default='/work');p.add_argument('--source',default='corpus.jsonl');p.add_argument('--steps',type=int,default=400);p.add_argument('--resume',action='store_true');a=p.parse_args()
    if a.action=='prepare': prepare(a.source,a.root)
    elif a.action=='train': train(a.root,a.steps,a.resume)
    else: promote(a.root)
