"""Optional real GPT-2 architecture training path; not run by Token Lab.
Python 3.11, torch 2.6.0, transformers 4.51.3, tokenizers 0.21.1.
Reads the SAME prepared dataset.json; trains its own byte BPE tokenizer.
A tiny random model demonstrates mechanics, not useful assistant capability.
"""
import argparse,json,os,random
from pathlib import Path
import torch
from tokenizers import ByteLevelBPETokenizer
from transformers import GPT2Config,GPT2LMHeadModel,PreTrainedTokenizerFast
p=argparse.ArgumentParser();p.add_argument('--root',default='/work');p.add_argument('--steps',type=int,default=200);p.add_argument('--resume',action='store_true');a=p.parse_args()
root=Path(a.root);ds=json.loads((root/'dataset.json').read_text());torch.manual_seed(42);random.seed(42)
# CPU default for broad compatibility; set DEVICE=cuda for a provisioned GPU.
device=os.environ.get('DEVICE','cpu');tokenizer=ByteLevelBPETokenizer()
tokenizer.train_from_iterator((r['text'] for r in ds['train']),vocab_size=512,min_frequency=2,special_tokens=['<bos>','<eos>','<pad>'])
tok=PreTrainedTokenizerFast(tokenizer_object=tokenizer._tokenizer,bos_token='<bos>',eos_token='<eos>',pad_token='<pad>')
config=GPT2Config(vocab_size=len(tok),n_positions=128,n_ctx=128,n_embd=128,n_layer=2,n_head=4,bos_token_id=tok.bos_token_id,eos_token_id=tok.eos_token_id,pad_token_id=tok.pad_token_id)
model=GPT2LMHeadModel(config).to(device);optimizer=torch.optim.AdamW(model.parameters(),lr=3e-4)
rows=[tok(r['text'],max_length=126,truncation=True)['input_ids']+[tok.eos_token_id] for r in ds['train']]
import hashlib
fingerprint=hashlib.sha256((root/'dataset.json').read_bytes()).hexdigest();cp=root/'transformer-checkpoint.pt';start=0
if a.resume and cp.exists():
    state=torch.load(cp,map_location='cpu',weights_only=True)
    if state['data_hash']!=fingerprint: raise ValueError('dataset mismatch')
    model.load_state_dict(state['model']);optimizer.load_state_dict(state['optimizer']);start=state['step'];torch.set_rng_state(state['cpu_rng']);random.setstate(state['python_rng'])
    if device.startswith('cuda'): torch.cuda.set_rng_state_all(state['cuda_rng'])
model.train()
for step in range(start,a.steps):
    chosen=random.choices(rows,k=4);width=max(map(len,chosen));ids=torch.full((4,width),tok.pad_token_id,dtype=torch.long,device=device);mask=torch.zeros_like(ids)
    for i,row in enumerate(chosen): ids[i,:len(row)]=torch.tensor(row,device=device);mask[i,:len(row)]=1
    labels=ids.clone();labels[mask==0]=-100
    optimizer.zero_grad(set_to_none=True);loss=model(input_ids=ids,attention_mask=mask,labels=labels).loss
    loss.backward();torch.nn.utils.clip_grad_norm_(model.parameters(),1);optimizer.step()
    if (step+1)%50==0 or step+1==a.steps:
        state={'model':model.state_dict(),'optimizer':optimizer.state_dict(),'step':step+1,'cpu_rng':torch.get_rng_state(),'python_rng':random.getstate(),'cuda_rng':torch.cuda.get_rng_state_all() if device.startswith('cuda') else [],'data_hash':fingerprint}
        tmp=root/'checkpoint.tmp';torch.save(state,tmp);os.replace(tmp,cp)
        print(json.dumps({'step':step+1,'training_loss':float(loss.detach())}))
export=root/'transformer-candidate';export.mkdir(exist_ok=True)
model.save_pretrained(export,safe_serialization=True);tok.save_pretrained(export)
# Candidate only: no release pointer is changed. Evaluate before promotion.
(export/'lineage.json').write_text(json.dumps({'data_sha256':fingerprint,'seed':42,'steps':a.steps,'status':'candidate-not-approved'}))
