Colorizer round6: smoke and initial model verified

experiments/region-coherence-20260928/SHA256SUMS.json

@@ -0,0 +1,18 @@
+{
+  "data_and_eval.py": "019ea992052f0233954b86e06e31c85c74ca313e67a55ff7fb03248f8874aaf5",
+  "initial/README.md": "2508e51f14898f651c49adb184d1570b8adad3702313dee9bdac6fa0390d2bec",
+  "initial/config.json": "eaeb49b39f9116bfed0d954e81852dc93476298752787dba88a63c0eec97ccec",
+  "initial/model.safetensors": "ec1f27d74533adc83f7ab3639a091fc4d8738a434dafc7d172c7873c28a9e715",
+  "protocol.json": "926776a1b4c2b29dbd292dc68cd68bd12df3fdc4db541044fdadca48c3a5e4e5",
+  "smoke.json": "d0dc2acce2f8273d58479f00a32a6618e3eecfb2220d00afc6721ddf7f08442c",
+  "source/data.py": "eb43b886fe45435841439c36646985f914386a48faa712fb6806accac2ad48e0",
+  "source/legacy_eval.py": "dd7e8bd5f961dfb6f1c2ce98fac41824c5009188e306000b195e922e4ed03e58",
+  "source/metrics.py": "af0ac996f6be0a351a55e372fab637a0d06740afe4a744cb80d0eae56ec97f66",
+  "source/model.py": "4cc57f82ffd6378bdf23a088ce7a1ed56c09e5673de3ee5232b8ee1cacc9be0a",
+  "source/objectives.py": "133e85b6f2914c0e3803cc26a26fa29b779a49036219dd245c2ecfe0ba411e20",
+  "source/persistence.py": "8b26dd8acdc135ecbfcdd30b7f7723feb6008ac820c083403da1e820bdc15d33",
+  "source/previous_manifest.json": "24bc081bf48f1109d9dfa217ae0d8593c0c06af6785a5cbfc5dc5d9e25461f5c",
+  "source/semantic_model.py": "b019358cbcaa214743cbd244d20eb6e59eeff0ab5264093ec0927be8529b25a2",
+  "source/spatial.py": "d13abe399ef049e21a6459a7003461afaae0c00e0c560262c6cf375de4c9884a",
+  "train.py": "8d7ceb77a742afd65c9ab7ddf58e56b86ff235900371ce57b4b01c65a759a4cb"
+}

experiments/region-coherence-20260928/data_and_eval.py

@@ -0,0 +1,234 @@
+import os,sys,json,random,time,hashlib,shutil,math,traceback,io
+from pathlib import Path
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch.utils.data import Dataset,DataLoader
+from huggingface_hub import hf_hub_download
+import pyarrow as pa
+import pyarrow.parquet as pq
+from PIL import Image,ImageDraw
+from skimage.color import rgb2lab,lab2rgb
+from skimage.segmentation import slic
+import cv2
+import persistence
+from semantic_model import load_semantic,save_semantic
+from objectives import color_loss
+from data import decode_image
+import legacy_eval as ev
+REPO='User-2468/mini-unet-colorizer'
+REV='7b10485ee9d205ce2f7b660dfc6a90012d78b1cb'
+OLD='experiments/round6-20260928'
+OUT=Path('/work/results');OUT.mkdir(parents=True,exist_ok=True)
+persistence.PREFIX='experiments/round7-20260928'
+ev.OUT=OUT
+START=time.monotonic()
+SEED=709
+STEPS=2000
+DEVICE='cpu' if os.environ.get('SMOKE')=='1' else 'cuda'
+ev.DEVICE=DEVICE
+DURABLE=None
+def write(name,obj):
+ p=OUT/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_text(json.dumps(obj,indent=2));return obj
+def deadline():
+ if time.monotonic()-START>9000:raise TimeoutError('Graceful deadline')
+ev.deadline=deadline
+def losses(p,t):
+ p=F.adaptive_avg_pool2d(p.float(),(64,64));t=t.float()
+ pixel=(p-t).abs().mean((1,2,3))/20
+ a=F.avg_pool2d(p,2).flatten(2);b=F.avg_pool2d(t,2).flatten(2)
+ angles=torch.arange(8,device=p.device)*math.pi/8
+ directions=torch.stack([angles.cos(),angles.sin()],1)
+ ap=torch.einsum('kc,bcn->bkn',directions,a).sort(-1).values
+ bp=torch.einsum('kc,bcn->bkn',directions,b).sort(-1).values
+ distribution=(ap-bp).abs().mean((1,2))/20
+ edge=((torch.diff(p,dim=-1)-torch.diff(t,dim=-1)).abs().mean((1,2,3))+(torch.diff(p,dim=-2)-torch.diff(t,dim=-2)).abs().mean((1,2,3)))/40
+ return pixel+.25*distribution+.1*edge
+def select_loss(p,teacher,original,mode):
+ a=losses(p,teacher)
+ if mode=='teacher':return a.mean(),torch.zeros_like(a,dtype=torch.bool)
+ b=losses(p,original);choose=b<a
+ return torch.where(choose,b,a).mean(),choose
+def smoke():
+ torch.set_num_threads(4);torch.manual_seed(SEED)
+ p=torch.randn(2,2,64,64,requires_grad=True);t=torch.randn_like(p)
+ assert torch.allclose(losses(p,t).mean(),color_loss(p,t)[0],atol=1e-6)
+ # Whole-image choice: exact red or blue is better than an average or a patchwork.
+ red=torch.zeros(1,2,64,64);red[:,0]=30
+ blue=-red;mix=red.clone();mix[:,:,:,32:]=blue[:,:,:,32:]
+ assert select_loss(red,red,blue,'set')[0]==0
+ assert select_loss(torch.zeros_like(red),red,blue,'set')[0]>0
+ assert select_loss(mix,red,blue,'set')[0]>0
+ m=load_semantic('/work/initial',DEVICE)
+ assert sum(p.numel() for p in m.parameters())==3994676
+ x=torch.rand(2,1,96,128,device=DEVICE)*2-1
+ pred=m(x);loss,_=select_loss(pred,t.to(DEVICE),-t.to(DEVICE),'set')
+ loss.backward()
+ assert torch.isfinite(loss) and all(torch.isfinite(p.grad).all() for p in m.parameters() if p.grad is not None)
+ save_semantic(m,OUT/'initial')
+ re=load_semantic(OUT/'initial',DEVICE)
+ with torch.no_grad():assert torch.allclose(m.eval()(x),re(x),atol=1e-5)
+ write('smoke.json',{'passed':True,'parameters':3994676,'loss_equivalence':True,'whole_image_choice':True,'finite_backward':True,'reload':True})
+ del m,re
+class Photos(Dataset):
+ def __init__(self,table,ids,cache,archive):
+  self.table,self.ids,self.cache,self.archive=table,ids,cache,archive;self.epoch=0
+ def __len__(self):return len(self.ids)
+ def __getitem__(self,k):
+  im=decode_image(self.table,self.ids[k]);L,ab,gray=ev.gray_arrays(im)
+  # Independent per-example RNG; augmentation differences never alter image ordering/flips.
+  rng=np.random.default_rng(np.random.SeedSequence([SEED,self.epoch,k]))
+  target=np.array(self.cache[k],np.float32)
+  if self.archive and rng.random()<.5:
+   rgb=np.asarray(im.resize((256,256),Image.Resampling.BILINEAR),np.float32)/255
+   weights=rng.dirichlet(np.array([3.,5.,2.]))
+   g=(rgb*weights).sum(-1)
+   g=np.clip((g**rng.uniform(.7,1.5)-.5)*rng.uniform(.6,1.1)+.5+rng.uniform(-.08,.08),0,1)
+   if rng.random()<.5:g=cv2.GaussianBlur(g,(5,5),float(rng.uniform(.3,1.2)))
+   if rng.random()<.6:g=np.clip(g+rng.normal(0,rng.uniform(.003,.025),g.shape),0,1)
+   L=(rgb2lab(np.repeat(g[...,None],3,-1).astype(np.float32))[...,0][None]/50-1).astype(np.float32)
+  else:
+   if rng.random()<.5:L=np.clip(L*rng.uniform(.7,1.15)+rng.uniform(-.12,.12),-1,1)
+  flip=np.random.default_rng(np.random.SeedSequence([SEED+1,self.epoch,k])).random()<.5
+  if flip:L=L[...,::-1].copy();ab=ab[...,::-1].copy();target=target[...,::-1].copy()
+  original=cv2.resize(ab.transpose(1,2,0),(64,64),interpolation=cv2.INTER_AREA).transpose(2,0,1).copy()
+  return torch.from_numpy(L),torch.from_numpy(target),torch.from_numpy(original)
+@torch.inference_mode()
+def predict(m,L):
+ x=torch.from_numpy(L)[None].to(DEVICE);raw=m(x)
+ guided=ev.guided_chroma(x,raw,8)
+ return raw,guided
+def toimage(L,ab):
+ rgb=lab2rgb(np.concatenate([(L[0]*50+50)[...,None],ab],-1))
+ return Image.fromarray(np.uint8(np.clip(rgb,0,1)*255))
+@torch.inference_mode()
+def review(m,table,ids,name):
+ m.eval();rows=[]
+ # Reference-derived surface regions for diagnostics only, never inference.
+ for i in ids:
+  L,target,_=ev.gray_arrays(decode_image(table,i))
+  raw,pred=predict(m,L);x=torch.from_numpy(L)[None].to(DEVICE)
+  resize=F.interpolate(x,(224,224),mode='bilinear',align_corners=False)
+  resized=F.interpolate(m(resize),(256,256),mode='bilinear',align_corners=False)
+  contrast=m((x*.95+.02).clamp(-1,1))
+  pab=pred[0].cpu().numpy().transpose(1,2,0)
+  rgb=np.asarray(decode_image(table,i).resize((256,256)),np.float32)/255
+  seg=slic(rgb,n_segments=100,compactness=10,start_label=0)
+  tab=target.transpose(1,2,0);variations=[]
+  for label in np.unique(seg):
+   mask=seg==label
+   if mask.sum()<64:continue
+   ground=tab[mask];center=np.median(ground,axis=0)
+   if np.percentile(np.linalg.norm(ground-center,axis=1),90)>5:continue
+   col=pab[mask];variations.append(float(np.mean(np.linalg.norm(col-np.median(col,axis=0),axis=1))))
+  rows.append({'id':int(i),'resize_ab_change':float((raw-resized).abs().mean()),'contrast_ab_change':float((raw-contrast).abs().mean()),'surface_chroma_variation':float(np.mean(variations)) if variations else None,'chroma_coverage':float((pred.norm(dim=1)>10).float().mean()),'mean_chroma':float(pred.norm(dim=1).mean())})
+ write(name+'_diagnostics.json',{'rows':rows,'summary':{k:float(np.mean([r[k] for r in rows if r[k] is not None])) for k in rows[0] if k!='id'},'warning':'Surface uniformity alone rewards grey collapse; never use alone to select model. Color differences from original are diagnostic only.'})
+ return rows
+@torch.inference_mode()
+def grid(models,table,ids,label):
+ order=np.random.default_rng(SEED+2).permutation(len(models))
+ names=list(models);key={chr(65+j):names[int(i)] for j,i in enumerate(order)}
+ write(label+'_key.json',key)
+ for page in range(0,min(len(ids),32),8):
+  chosen=ids[page:page+8];canvas=Image.new('RGB',(256*(len(models)+1),274*len(chosen)),'white');draw=ImageDraw.Draw(canvas)
+  for row,index in enumerate(chosen):
+   L,_,gray=ev.gray_arrays(decode_image(table,index));draw.text((2,row*274),f'{index}: gray | '+' | '.join(key),fill='black')
+   canvas.paste(Image.fromarray(np.uint8(gray*255)),(0,row*274+18))
+   for j,i in enumerate(order):
+    _,p=predict(models[names[int(i)]],L)
+    canvas.paste(toimage(L,p[0].cpu().numpy().transpose(1,2,0)),((j+1)*256,row*274+18))
+  canvas.save(OUT/f'{label}_{page//8}.jpg',quality=92)
+def main():
+ global DURABLE
+ torch.set_num_threads(6);torch.manual_seed(SEED);random.seed(SEED);np.random.seed(SEED)
+ DURABLE=persistence.DurableRun(OUT)
+ shutil.copy2('/work/runner.py',OUT/'runner.py');shutil.copy2('/work/PROTOCOL.md',OUT/'PROTOCOL.md')
+ shutil.copytree('/work/source',OUT/'source',ignore=shutil.ignore_patterns('__pycache__'),dirs_exist_ok=True)
+ smoke();DURABLE.sync('round7 initial checkpoint and smoke verified')
+ if DEVICE=='cpu':return
+ torch.backends.cudnn.benchmark=True
+ tables=[]
+ for repo,rev,files in [('johnowhitaker/imagenette2-320','771c1310a2487e8076ede6b7d6307244aa8400af',['default/train/0000.parquet']),('detection-datasets/coco','26ddc382fe75dfc2a0655b5977e296ea10efebce',['default/train/0000.parquet','default/train/0001.parquet'])]:
+  for f in files:tables.append(pq.read_table(hf_hub_download(repo,f,repo_type='dataset',revision=rev),columns=['image']))
+ table=pa.concat_tables(tables);manifest=json.loads(Path('/work/source/previous_manifest.json').read_text());ids=manifest['train']
+ cache=np.empty((len(ids),2,64,64),np.float16)
+ sums=json.loads(Path(hf_hub_download(REPO,OLD+'/SHA256SUMS.json',revision=REV)).read_text())
+ for start in range(0,len(ids),2048):
+  f=f'teacher_cache/{start:05d}.npz';path=Path(hf_hub_download(REPO,OLD+'/'+f,revision=REV))
+  assert hashlib.sha256(path.read_bytes()).hexdigest()==sums[f]
+  with np.load(path) as z:
+   assert z['indices'].tolist()==ids[start:start+len(z['indices'])]
+   cache[start:start+len(z['target'])]=z['target']
+ fresh=pq.read_table(hf_hub_download('detection-datasets/coco','default/val/0001.parquet',repo_type='dataset',revision='26ddc382fe75dfc2a0655b5977e296ea10efebce'),columns=['image'])
+ hashes={hashlib.sha256(table['image'][int(i)].as_py()['bytes']).digest() for i in ids}
+ freshids=[]
+ for i in np.random.default_rng(SEED).permutation(len(fresh)):
+  if hashlib.sha256(fresh['image'][int(i)].as_py()['bytes']).digest() not in hashes:freshids.append(int(i))
+  if len(freshids)==160:break
+ write('manifest.json',{'train':ids,'development':freshids[:80],'heldout':freshids[80:],'fresh_file':'default/val/0001.parquet','revision':'26ddc382fe75dfc2a0655b5977e296ea10efebce','prior_model_revision':REV,'upstream_pretraining_overlap_unknown':True,'near_duplicates_not_excluded':True})
+ DURABLE.sync('data cache verified and split fixed')
+ m=load_semantic('/work/initial',DEVICE)
+ write('initial_validation.json',ev.evaluate(m,table,manifest['validation']))
+ review(m,fresh,freshids[:80],'initial_development')
+ review(m,fresh,freshids[80:],'initial_heldout')
+ write('initial_heldout_reference_metrics.json',ev.evaluate(m,fresh,freshids[80:]))
+ del m
+ arms=[('teacher_clean','teacher',False),('set_clean','set',False),('teacher_archive','teacher',True),('set_archive','set',True)]
+ for name,mode,archive in arms:
+  torch.manual_seed(SEED);np.random.seed(SEED);random.seed(SEED)
+  m=load_semantic('/work/initial',DEVICE)
+  opt=torch.optim.AdamW([{'params':m.encoder.parameters(),'lr':5e-6},{'params':[p for n,p in m.named_parameters() if not n.startswith('encoder.')],'lr':5e-5}],weight_decay=.01)
+  sched=torch.optim.lr_scheduler.LambdaLR(opt,lambda step:min((step+1)/100,1)*(.2+.8*(1+math.cos(math.pi*min(step,STEPS)/STEPS))/2))
+  ds=Photos(table,ids,cache,archive)
+  loader=DataLoader(ds,batch_size=32,shuffle=True,num_workers=4,pin_memory=True,generator=torch.Generator().manual_seed(SEED))
+  it=iter(loader);history=[];choices=0;seen=0
+  for step in range(1,STEPS+1):
+   deadline();m.train()
+   for layer in m.encoder.modules():
+    if isinstance(layer,torch.nn.BatchNorm2d):layer.eval()
+   try:L,t,o=next(it)
+   except StopIteration:ds.epoch+=1;it=iter(loader);L,t,o=next(it)
+   L,t,o=L.to(DEVICE),t.to(DEVICE),o.to(DEVICE);opt.zero_grad(set_to_none=True)
+   with torch.autocast('cuda',dtype=torch.bfloat16):p=m(L)
+   loss,choice=select_loss(p,t,o,mode)
+   assert torch.isfinite(loss)
+   loss.backward();torch.nn.utils.clip_grad_norm_(m.parameters(),1,error_if_nonfinite=True);opt.step();sched.step()
+   choices+=int(choice.sum());seen+=len(L)
+   if step%100==0:
+    history.append({'step':step,'loss':float(loss),'original_choice_fraction':choices/seen})
+    print('STEP',name,step,round(float(loss),4),flush=True)
+   if step%500==0:
+    path=OUT/name/f'step_{step:05d}';save_semantic(m,path)
+    torch.save({'optimizer':opt.state_dict(),'scheduler':sched.state_dict(),'step':step,'epoch':ds.epoch,'torch_rng':torch.get_rng_state(),'cuda_rng':torch.cuda.get_rng_state_all(),'numpy_rng':np.random.get_state(),'python_rng':random.getstate()},path/'training_state.pt')
+    write(name+'/history.json',history);write('status.json',{'status':'training','arm':name,'step':step,'elapsed_seconds':time.monotonic()-START})
+    DURABLE.sync(name+' checkpoint '+str(step))
+  write(name+'_validation.json',ev.evaluate(m,table,manifest['validation']))
+  review(m,fresh,freshids[:80],name+'_development')
+  review(m,fresh,freshids[80:],name+'_heldout')
+  write(name+'_heldout_reference_metrics.json',ev.evaluate(m,fresh,freshids[80:]))
+  ev.probes(m,name)
+  DURABLE.sync(name+' complete diagnostics')
+  del m,opt,sched,it,loader;torch.cuda.empty_cache()
+ models={'round6':load_semantic('/work/initial',DEVICE)}
+ for name,_,_ in arms:models[name]=load_semantic(OUT/name/f'step_{STEPS:05d}',DEVICE)
+ grid(models,fresh,freshids[:80],'blinded_development')
+ grid(models,fresh,freshids[80:],'blinded_heldout')
+ # Familiar historical development probes; never claim these are a held-out archival benchmark.
+ import requests
+ sources={'migrant_mother':'https://cdn.loc.gov/service/pnp/ppmsca/50200/50236v.jpg','power_house_mechanic':'https://www.archives.gov/exhibits/picturing_the_century/images/port_hine_022_v36.jpg'}
+ for name,url in sources.items():
+  try:
+   response=requests.get(url,timeout=45);response.raise_for_status()
+   im=Image.open(io.BytesIO(response.content)).convert('RGB')
+   tb=pa.table({'image':[{'bytes':response.content,'path':None}]})
+   grid(models,tb,[0],name)
+   write(name+'_source.json',{'url':url,'sha256':hashlib.sha256(response.content).hexdigest(),'use':'previously seen archival development probe; original colors unknown'})
+  except Exception as e:write(name+'_error.json',{'type':type(e).__name__,'message':str(e)})
+ write('status.json',{'status':'completed','elapsed_seconds':time.monotonic()-START,'production_approved':False,'next':'Review blinded grids for coherent plausible color and gray/sepia collapse. This pilot does not test a stochastic conditioned decoder.'})
+ DURABLE.sync('round7 four-arm experiment completed')
+if __name__=='__main__':
+ try:main()
+ except BaseException as e:
+  write('failure.json',{'type':type(e).__name__,'traceback':traceback.format_exc().replace(os.environ.get('HF_TOKEN','__EMPTY__'),'[REDACTED]')})
+  if DURABLE:DURABLE.sync('round7 failure report and latest saved checkpoints')
+  raise

experiments/region-coherence-20260928/initial/README.md

@@ -0,0 +1,5 @@
+# Experimental semantic colorizer
+
+Head: palette. Parameters: 3994676.
+
+Not approved for production. Requires semantic_model.py; incompatible with the old U-Net loader. Input is Lab lightness normalized to [-1,1]; output is Lab ab. See the run protocol, provenance, selection and visual comparisons. Predictions are plausible colors, not recovered historical truth.

experiments/region-coherence-20260928/initial/config.json

@@ -0,0 +1,7 @@
+{
+  "architecture": "SemanticColorizer",
+  "head": "palette",
+  "width": 128,
+  "queries": 16,
+  "format_version": 1
+}

experiments/region-coherence-20260928/initial/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec1f27d74533adc83f7ab3639a091fc4d8738a434dafc7d172c7873c28a9e715
+size 16112432

experiments/region-coherence-20260928/protocol.json

@@ -0,0 +1,37 @@
+{
+  "base_revision": "1a9eb8af2754ad2329a24cfe50d388cb559441d0",
+  "artifact_revision": "a78fd9de52e4e3884e4530f634754d2a910a2c55",
+  "teacher_cache_revision": "7b10485ee9d205ce2f7b660dfc6a90012d78b1cb",
+  "seed": 92851,
+  "generator_parameters": 3994676,
+  "steps_per_arm": 2000,
+  "batch_size": 32,
+  "arms": [
+    "control",
+    "region"
+  ],
+  "matched": "Same fixed selected targets, input augmentation batches, initial weights, optimizer and updates. Treatment adds .4 object residual consistency + .15 local missing chroma.",
+  "supervised": "L1/20 + .25 sliced chroma distribution + .1 gradient difference (existing r.losses)",
+  "target_choice": "One complete original or cached DDColor target per training image, selected once by v3 complete-image loss. No per-pixel target mixing. Reject teacher proposal if it removes over40% of original chroma or has >1.5x+1.5 mean chroma edge strength.",
+  "region_loss": "Mean absolute spatial residual after subtracting regional mean prediction-target error. Allows constant alternative hue shifts while retaining target patterns.",
+  "segmentation": "Frozen torchvision DeepLabV3 ResNet50 COCO_WITH_VOC_LABELS_V1 on original RGB, default520px preprocessing. Confident connected foreground interiors + homogeneous original-color SLIC background surfaces. Max64 regions/image.",
+  "inference": "Unchanged architecture and one-pass inference; segmentation/teacher are training-only.",
+  "data": "Seeded8192 subset of original training fold; pinned Imagenette/COCO shards and verified cached DDColor. 50% mild archival-style input degradations.",
+  "evaluation": "Reused dense-run development and test are explicitly regression sets. New128 COCO val/0001 examples exclude round7 and dense-run evaluation IDs and training byte duplicates. New holdout used only after training; familiar archived-photo probes are not independent historical validation.",
+  "limitations": [
+    "20 foreground semantic classes only",
+    "touching same-class objects can merge",
+    "RGB-derived masks are imperfect",
+    "DDColor targets are fallible",
+    "teacher/encoder upstream pretraining overlap unknown",
+    "one seed and short pilot",
+    "synthetic film variation is not true archival training",
+    "frozen target selection can retain v3 biases"
+  ],
+  "budget": {
+    "hard_timeout_minutes": 30,
+    "training_stop_minutes": 23,
+    "maximum_compute_usd": 1.25
+  },
+  "release_policy": "All checkpoints retained for review; no automatic production replacement"
+}

experiments/region-coherence-20260928/smoke.json

@@ -0,0 +1,10 @@
+{
+  "passed": true,
+  "generator_parameters": 3994676,
+  "uniform_shift_allowed": true,
+  "patch_penalized": true,
+  "gray_collapse_penalized": true,
+  "pattern_preserved": true,
+  "finite_backward": true,
+  "checkpoint_roundtrip": true
+}

experiments/region-coherence-20260928/source/data.py

@@ -0,0 +1,116 @@
+"""Fixed color-bin semantics and reproducible dataset membership."""
+import hashlib
+import io
+import json
+from pathlib import Path
+import numpy as np
+from PIL import Image
+import pyarrow.parquet as pq
+from scipy.spatial import cKDTree
+from skimage.color import rgb2lab
+import torch
+from torch.utils.data import Dataset
+
+class ColorBins:
+    def __init__(self, centers, weights=None):
+        self.centers = np.asarray(centers, np.float32)
+        if self.centers.ndim != 2 or self.centers.shape[1] != 2:
+            raise ValueError("Expected Q by 2 color centers")
+        self.tree = cKDTree(self.centers)
+        self.weights = np.ones(len(self.centers), np.float32) if weights is None else np.asarray(weights,np.float32)
+
+    def encode(self, ab, k=5, sigma=5):
+        k = min(k, len(self.centers))
+        if sigma <= 0 or k < 1:
+            raise ValueError("Invalid encoding parameters")
+        dist, idx = self.tree.query(ab.reshape(-1,2), k=k)
+        dist, idx = dist.reshape(-1,k), idx.reshape(-1,k)
+        logw = -dist**2/(2*sigma**2)
+        logw -= logw.max(1, keepdims=True)
+        weight = np.exp(logw); weight /= weight.sum(1,keepdims=True)
+        shape = (*ab.shape[:-1], k)
+        return idx.reshape(shape).astype(np.int64), weight.reshape(shape).astype(np.float32)
+
+def load_table(path):
+    table = pq.read_table(path)
+    return table
+
+def validate_manifest(path, table, manifest):
+    with open(path,'rb') as f:
+        digest=hashlib.file_digest(f,'sha256').hexdigest()
+    if digest!=manifest['dataset_sha256'] or len(table)!=manifest['rows']:
+        raise ValueError('Dataset differs from pinned split manifest')
+    groups=[list(map(int,manifest[k])) for k in ['train','validation','test']]
+    if any(not g or len(g)!=len(set(g)) or min(g)<0 or max(g)>=len(table) for g in groups):
+        raise ValueError('Invalid or repeated indices in manifest')
+    if any(set(groups[i])&set(groups[j]) for i,j in [(0,1),(0,2),(1,2)]):
+        raise ValueError('Train/validation/test overlap')
+
+def decode_image(table, index):
+    obj = table['image'][int(index)].as_py()
+    return Image.open(io.BytesIO(obj['bytes'])).convert('RGB')
+
+def legacy_split(n, seed=0):
+    # Exactly reproduce datasets.Dataset.train_test_split(test_size=.05, seed=0).
+    order = np.random.default_rng(seed).permutation(n)
+    nval = int(np.ceil(.05*n))
+    return order[nval:], order[:nval]
+
+def stratified_subset(indices, labels, per_class, seed):
+    rng = np.random.default_rng(seed)
+    selected = []
+    for label in sorted(set(labels)):
+        group = np.asarray([i for i in indices if labels[i] == label])
+        selected.extend(rng.choice(group, min(per_class,len(group)), replace=False).tolist())
+    return np.asarray(selected, dtype=np.int64)
+
+class PhotoDataset(Dataset):
+    def __init__(self, table, indices, bins, size=256, augment=False):
+        self.table, self.indices, self.bins = table, list(map(int,indices)), bins
+        self.size, self.augment = size, augment
+
+    def __len__(self):
+        return len(self.indices)
+
+    def __getitem__(self, item):
+        # Keep square preprocessing for controlled comparison with historical runs.
+        # Production inference preserves aspect ratio separately.
+        image = decode_image(self.table,self.indices[item]).resize((self.size,self.size),Image.Resampling.BILINEAR)
+        arr = np.asarray(image,dtype=np.float32)/255
+        if self.augment and torch.rand(()) < .5:
+            arr = arr[:,::-1].copy()
+        lab = rgb2lab(arr).astype(np.float32)
+        idx, weight = self.bins.encode(lab[...,1:])
+        return (torch.from_numpy((lab[...,:1]/50-1).transpose(2,0,1).copy()),
+                torch.from_numpy(lab[...,1:].transpose(2,0,1).copy()),
+                torch.from_numpy(idx),torch.from_numpy(weight))
+
+def estimate_weights(table, indices, bins, mix=.7, limit=1000, seed=0):
+    """Re-estimate the prior ON checkpoint bins; never replace the bin vocabulary."""
+    if not 0 <= mix <= 1:
+        raise ValueError("Rebalance mixture must be in [0,1]")
+    rng=np.random.default_rng(seed)
+    chosen=rng.choice(indices,min(limit,len(indices)),replace=False)
+    counts=np.ones(len(bins.centers),np.float64)*1e-3
+    for index in chosen:
+        image=decode_image(table,index).resize((32,32))
+        ab=rgb2lab(np.asarray(image,dtype=np.float32)/255)[...,1:]
+        # Low chroma is a heuristic, not proof that an image was originally B&W.
+        if np.linalg.norm(ab,axis=-1).mean()<3:
+            continue
+        idx,weight=bins.encode(ab)
+        np.add.at(counts,idx.ravel(),weight.ravel())
+    prior=counts/counts.sum()
+    weights=1/((1-mix)*prior+mix/len(prior))
+    weights/=np.sum(prior*weights)
+    return weights.astype(np.float32),prior.astype(np.float32)
+
+def write_manifest(path, table, train, validation, test, dataset_sha256):
+    groups=[set(map(int,g)) for g in [train,validation,test]]
+    assert not groups[0]&groups[1] and not groups[0]&groups[2] and not groups[1]&groups[2]
+    content={'dataset':'johnowhitaker/imagenette2-320','dataset_sha256':dataset_sha256,
+        'rows':len(table),'split_seed':0,'legacy_holdout_fraction':.05,
+        'train':list(map(int,train)),'validation':list(map(int,validation)), 'test':list(map(int,test)),
+        'caveat':'Matches the recent seed-0 holdout; older upstream training exposure is not established.'}
+    Path(path).write_text(json.dumps(content,indent=2))
+    return content

experiments/region-coherence-20260928/source/legacy_eval.py

@@ -0,0 +1,150 @@
+"""Round 4: matched multiscale structure and DDColor distillation experiments."""
+import os,sys,json,time,random,hashlib,traceback,shutil
+from pathlib import Path
+import numpy as np
+import cv2
+from PIL import Image,ImageDraw
+from skimage import data as sample_data
+from skimage.color import rgb2lab,lab2rgb
+import pyarrow as pa
+import pyarrow.parquet as pq
+import torch
+import torch.nn.functional as F
+from torch.utils.data import Dataset,DataLoader
+from huggingface_hub import HfApi,hf_hub_download,PyTorchModelHubMixin
+from model import load_model,save_model
+from data import decode_image,ColorBins
+from metrics import soft_ce,per_image_metrics
+from spatial import guided_chroma
+
+OUT=Path('round6');OUT.mkdir(exist_ok=True)
+DURABLE=None
+START=time.monotonic();DEADLINE=START+3.5*3600
+DEVICE='cuda';SEED=409;STEPS=3000
+BASE_REV='704fa80d792c3d759db91daa00b2dcfe6f0f6412'
+BASE_HASH='0e4c417375684a044860f8af3ac3a2fb44e1a5729254ca33abda758ea71aea6e'
+TEACHER_ID='piddnad/ddcolor_artistic'
+torch.set_num_threads(6)
+
+def deadline():
+    if time.monotonic()>DEADLINE:raise TimeoutError('Graceful export before remote hard timeout')
+
+def gray_arrays(im,style='gray'):
+    im=im.resize((256,256),Image.Resampling.BILINEAR)
+    rgb=np.asarray(im,np.float32)/255
+    lab=rgb2lab(rgb).astype(np.float32)
+    if style=='film':
+        g=(rgb*np.array([.42,.45,.13],np.float32)).sum(-1)
+        g=np.clip((g**1.15-.5)*.8+.5,0,1)
+    else:g=np.asarray(im.convert('L'),np.float32)/255
+    # Feed the same neutral sRGB input to teacher and student.
+    gray=np.repeat(g[...,None],3,-1).astype(np.float32)
+    L=rgb2lab(gray)[...,0].astype(np.float32)
+    return (L[None]/50-1).copy(),lab[...,1:].transpose(2,0,1).copy(),gray
+
+class Photos(Dataset):
+    def __init__(self,table,ids,bins=None,cache=None,augment=False,style='gray'):
+        self.table,self.ids,self.bins,self.cache,self.augment,self.style=table,ids,bins,cache,augment,style
+    def __len__(self):return len(self.ids)
+    def __getitem__(self,k):
+        L,ab,gray=gray_arrays(decode_image(self.table,self.ids[k]),self.style)
+        teach=np.array(self.cache[k],np.float32) if self.cache is not None else np.zeros((2,64,64),np.float32)
+        if self.augment:
+            if random.random()<.5:L=L[...,::-1].copy();ab=ab[...,::-1].copy();teach=teach[...,::-1].copy()
+            if random.random()<.3:L=np.clip(L*random.uniform(.85,1.15)+random.uniform(-.08,.08),-1,1)
+        if self.bins is not None:
+            idx,w=self.bins.encode(ab.transpose(1,2,0));return torch.from_numpy(L),torch.from_numpy(ab),torch.from_numpy(idx),torch.from_numpy(w),torch.from_numpy(teach)
+        return torch.from_numpy(L),torch.from_numpy(ab),torch.from_numpy(gray.transpose(2,0,1)),self.ids[k]
+
+def patch_excess(p,t):
+    """Excess color differences across 4–64px regions, in target-flat areas."""
+    values=[]
+    for scale in [4,16]:
+        a=F.avg_pool2d(p,scale);b=F.avg_pool2d(t,scale)
+        for offset in [1,4]:
+            for dim in [-1,-2]:
+                x=a.narrow(dim,offset,a.shape[dim]-offset)-a.narrow(dim,0,a.shape[dim]-offset)
+                y=b.narrow(dim,offset,b.shape[dim]-offset)-b.narrow(dim,0,b.shape[dim]-offset)
+                dx=x.norm(dim=1);dy=y.norm(dim=1);mask=(dy<3).float()
+                values.append(((dx-dy-1).relu()*mask).sum((1,2))/mask.sum((1,2)).clamp_min(1))
+    return sum(values)/len(values)
+
+def structural_loss(p,t):
+    terms=[]
+    for scale in [4,16]:
+        a=F.avg_pool2d(p,scale);b=F.avg_pool2d(t,scale)
+        for offset in [1,4]:
+            for dim in [-1,-2]:
+                x=a.narrow(dim,offset,a.shape[dim]-offset)-a.narrow(dim,0,a.shape[dim]-offset)
+                y=b.narrow(dim,offset,b.shape[dim]-offset)-b.narrow(dim,0,b.shape[dim]-offset)
+                terms.append(F.smooth_l1_loss(x/10,y/10,beta=.3))
+    return sum(terms)/len(terms)
+
+def additional_losses(p,t,teacher):
+    small=F.avg_pool2d(p,4);target=F.avg_pool2d(t,4)
+    # Teacher is fallible; reduce guidance when it conflicts strongly with ground truth.
+    confidence=torch.exp(-(teacher-target).norm(dim=1,keepdim=True)/30).detach()
+    kd=(F.smooth_l1_loss(small/20,teacher/20,beta=.5,reduction='none')*confidence).mean()
+    neutral=(t.norm(dim=1)<3).float()
+    neutral_loss=(p.norm(dim=1)*neutral).sum()/neutral.sum().clamp_min(1)/20
+    return structural_loss(p,t),kd,neutral_loss
+
+def measure(L,p,t,raw=None):
+    r=per_image_metrics(L,p,t);C=p.norm(dim=1);T=t.norm(dim=1)
+    for name,mask,event in [('missed_color',T>12,C<5),('neutral_spill',T<3,C>10)]:
+        r[name]=(mask&event).sum((1,2))/mask.sum((1,2)).clamp_min(1)
+    r['color_coverage']=(C>10).float().mean((1,2));r['patch_excess']=patch_excess(p,t)
+    r['raw_patch_excess']=patch_excess(p if raw is None else raw,t)
+    return r
+
+@torch.inference_mode()
+def teacher_predict(teacher,gray):
+    gray=F.interpolate(gray.to(DEVICE),size=(512,512),mode='bilinear',align_corners=False)
+    # Full precision is deliberate: spectral normalization / attention are not assumed BF16-safe.
+    out=teacher(gray).float()
+    assert out.shape[1]==2 and torch.isfinite(out).all()
+    return F.interpolate(out,size=(256,256),mode='bilinear',align_corners=False)
+
+@torch.inference_mode()
+def evaluate(model,table,ids,is_teacher=False):
+    result={};model.eval()
+    for style in ['gray','film']:
+        rows=[]
+        for L,t,gray,indices in DataLoader(Photos(table,ids,style=style),batch_size=2 if is_teacher else 8,num_workers=4):
+            deadline();L,t=L.to(DEVICE),t.to(DEVICE)
+            raw=teacher_predict(model,gray) if is_teacher else model.decode(model(L),.38)
+            pred=raw if is_teacher else guided_chroma(L,raw,8)
+            metrics=measure(L,pred,t,raw)
+            for j,index in enumerate(indices):rows.append({'index':int(index)}|{k:float(v[j]) for k,v in metrics.items()})
+        keep=[x for x in rows if x['target_chroma']>=5]
+        result[style]={'n_total':len(rows),'n_color':len(keep),'summary':{k:float(np.mean([x[k] for x in keep])) for k in keep[0] if k!='index'},'per_image':rows}
+    return result
+
+def selection(result,baseline):
+    scores=[];eligible=True
+    for style in ['gray','film']:
+        s=result[style]['summary'];b=baseline[style]['summary']
+        eligible &= (s['ab_error']<=b['ab_error']*1.05 and s['neutral_spill']<=b['neutral_spill']+.01
+                     and s['missed_color']<=b['missed_color']+.01 and s['color_coverage']>=.95*b['color_coverage']
+                     and s['patch_excess']<.85*b['patch_excess'] and s['raw_patch_excess']<.9*b['raw_patch_excess'])
+        scores.append(s['patch_excess']/max(b['patch_excess'],1e-6)+.3*s['neutral_spill']+.2*s['ab_error']/b['ab_error'])
+    return float(np.mean(scores)),bool(eligible)
+
+@torch.inference_mode()
+def probes(model,tag,teacher=False):
+    names=['astronaut','coffee','chelsea','rocket','camera','coins','moon'];images=[]
+    for name in names:
+        original=Image.fromarray(getattr(sample_data,name)()).convert('RGB')
+        L,_,gray=gray_arrays(original);x=torch.from_numpy(L)[None].to(DEVICE)
+        ab=teacher_predict(model,torch.from_numpy(gray.transpose(2,0,1))[None]) if teacher else guided_chroma(x,model.decode(model(x),.38),8)
+        lab=np.concatenate([(L[0]*50+50)[...,None],ab[0].cpu().numpy().transpose(1,2,0)],-1)
+        rgb=np.uint8(np.clip(lab2rgb(lab),0,1)*255)
+        images.append((name,Image.fromarray(np.uint8(gray*255)),Image.fromarray(rgb)))
+    canvas=Image.new('RGB',(512,len(names)*280),'white');d=ImageDraw.Draw(canvas)
+    for i,(name,gray,col) in enumerate(images):
+        canvas.paste(gray,(0,i*280+24));canvas.paste(col,(256,i*280+24));d.text((4,i*280+4),name+' input | '+tag,fill='black')
+    canvas.save(OUT/(tag+'.jpg'),quality=90)
+
+def write(name,obj):
+    (OUT/name).write_text(json.dumps(obj,indent=2));return obj
+

experiments/region-coherence-20260928/source/metrics.py

@@ -0,0 +1,35 @@
+"""Metrics are fidelity/artifact proxies; none certifies plausible color by itself."""
+import numpy as np
+import torch
+
+def soft_ce(logits, idx, weights, class_weights=None):
+    # FP32 reduction under autocast; no full target distribution materialized.
+    logits=logits.float()
+    idx=idx.permute(0,3,1,2)
+    weights=weights.permute(0,3,1,2)
+    nll=-((logits.gather(1,idx)-logits.logsumexp(1,keepdim=True))*weights).sum(1)
+    if class_weights is not None:
+        nll=nll*class_weights[idx[:,0]]
+    return nll.mean()
+
+def per_image_metrics(L, pred, target):
+    error=(pred-target).square().sum(1).sqrt().mean((1,2))
+    chroma=pred.square().sum(1).sqrt().mean((1,2))
+    true_chroma=target.square().sum(1).sqrt().mean((1,2))
+    seams=[]
+    for dim in [-1,-2]:
+        dp=pred.diff(dim=dim).square().sum(1).sqrt()
+        dt=target.diff(dim=dim).square().sum(1).sqrt()
+        dl=L.diff(dim=dim).abs()[:,0]*50
+        # Penalize excess chroma discontinuity where luminance is flat.
+        mask=(dl<2).float()
+        seams.append(((dp-dt).relu()*mask).sum((1,2))/mask.sum((1,2)).clamp_min(1))
+    return {'ab_error':error,'chroma':chroma,'target_chroma':true_chroma,
+            'excess_chroma_edge':sum(seams)/2}
+
+def summarize(rows):
+    keys=['ce','ab_error','chroma','target_chroma','excess_chroma_edge']
+    result={k:float(np.mean([r[k] for r in rows])) for k in keys}
+    result['n']=len(rows)
+    result['chroma_ratio']=result['chroma']/max(result['target_chroma'],1e-9)
+    return result

experiments/region-coherence-20260928/source/model.py

@@ -0,0 +1,122 @@
+"""Shared, checkpoint-compatible Mini U-Net. Parameters remain under 4M."""
+import json
+import math
+from pathlib import Path
+import numpy as np
+import torch
+from torch import nn
+import torch.nn.functional as F
+from huggingface_hub import PyTorchModelHubMixin, snapshot_download
+from safetensors.torch import load_file, save_file
+def double_conv(in_ch, out_ch):
+    return nn.Sequential(
+        nn.Conv2d(in_ch, out_ch, 3, padding=1, bias=False),
+        nn.BatchNorm2d(out_ch),
+        nn.ReLU(inplace=True),
+        nn.Conv2d(out_ch, out_ch, 3, padding=1, bias=False),
+        nn.BatchNorm2d(out_ch),
+        nn.ReLU(inplace=True),
+    )
+
+
+class DilatedContextBlock(nn.Module):
+    def __init__(self, channels, mid_ch=96, dilations=(2, 4, 8)):
+        super().__init__()
+        self.proj_in = nn.Sequential(
+            nn.Conv2d(channels, mid_ch, 1, bias=False),
+            nn.BatchNorm2d(mid_ch),
+            nn.ReLU(inplace=True),
+        )
+        layers = []
+        for d in dilations:
+            layers += [
+                nn.Conv2d(mid_ch, mid_ch, 3, padding=d, dilation=d, bias=False),
+                nn.BatchNorm2d(mid_ch),
+                nn.ReLU(inplace=True),
+            ]
+        self.dilated = nn.Sequential(*layers)
+        self.proj_out = nn.Sequential(
+            nn.Conv2d(mid_ch, channels, 1, bias=False),
+            nn.BatchNorm2d(channels),
+        )
+        nn.init.zeros_(self.proj_out[-1].weight)
+        self.relu = nn.ReLU(inplace=True)
+
+    def forward(self, x):
+        y = self.proj_in(x)
+        y = self.dilated(y)
+        y = self.proj_out(y)
+        return self.relu(x + y)
+
+
+class SmallUNetColorizer(
+    nn.Module,
+    PyTorchModelHubMixin,
+    pipeline_tag="image-to-image",
+    license="apache-2.0",
+    tags=["colorization", "unet", "image-to-image", "classification"],
+):
+    def __init__(self, bin_centers, in_ch: int = 1, base: int = 44,
+                 context_mid_ch: int = 96, context_dilations=(2, 4, 8)):
+        super().__init__()
+        self.in_ch, self.base = in_ch, base
+        num_bins = len(bin_centers)
+        self.num_bins = num_bins
+        self.register_buffer("bin_centers", torch.tensor(bin_centers, dtype=torch.float32))
+
+        self.enc1 = double_conv(in_ch, base)
+        self.enc2 = double_conv(base, base * 2)
+        self.enc3 = double_conv(base * 2, base * 4)
+        self.enc4 = double_conv(base * 4, base * 8)
+        self.pool = nn.MaxPool2d(2)
+        self.context = DilatedContextBlock(base * 8, mid_ch=context_mid_ch,
+                                            dilations=tuple(context_dilations))
+        self.up3 = nn.ConvTranspose2d(base * 8, base * 4, 2, stride=2)
+        self.dec3 = double_conv(base * 8, base * 4)
+        self.up2 = nn.ConvTranspose2d(base * 4, base * 2, 2, stride=2)
+        self.dec2 = double_conv(base * 4, base * 2)
+        self.up1 = nn.ConvTranspose2d(base * 2, base, 2, stride=2)
+        self.dec1 = double_conv(base * 2, base)
+        self.out_conv = nn.Conv2d(base, num_bins, 1)
+
+    def forward(self, x):
+        h, w = x.shape[-2:]
+        x = F.pad(x, (0, (-w) % 8, 0, (-h) % 8), mode="replicate")
+        e1 = self.enc1(x)
+        e2 = self.enc2(self.pool(e1))
+        e3 = self.enc3(self.pool(e2))
+        e4 = self.context(self.enc4(self.pool(e3)))
+        d3 = self.dec3(torch.cat([self.up3(e4), e3], dim=1))
+        d2 = self.dec2(torch.cat([self.up2(d3), e2], dim=1))
+        d1 = self.dec1(torch.cat([self.up1(d2), e1], dim=1))
+        return self.out_conv(d1)[..., :h, :w]
+
+    def decode(self, logits, temperature: float = 0.38):
+        if not math.isfinite(temperature) or temperature <= 0:
+            raise ValueError("temperature must be finite and positive")
+        probs_t = F.softmax(logits.float() / temperature, dim=1)
+        return torch.einsum("bqhw,qc->bchw", probs_t, self.bin_centers)
+
+
+def load_model(source, revision=None, device="cpu"):
+    path = Path(source)
+    if not path.is_dir():
+        path = Path(snapshot_download(source, revision=revision,
+            allow_patterns=["config.json", "model.safetensors"]))
+    config = json.loads((path / "config.json").read_text())
+    state = load_file(str(path / "model.safetensors"))
+    centers = torch.tensor(config["bin_centers"], dtype=torch.float32)
+    if not torch.equal(centers, state["bin_centers"]):
+        raise ValueError("Checkpoint config and state color bins differ; refusing ambiguous decode")
+    model = SmallUNetColorizer(**config)
+    model.load_state_dict(state, strict=True)
+    model.to(device).eval()
+    return model
+
+def save_model(model, path):
+    path = Path(path); path.mkdir(parents=True, exist_ok=True)
+    model.save_pretrained(path)
+    # Mixin config can retain constructor bins; use the actual authoritative buffer.
+    cfg = json.loads((path / "config.json").read_text())
+    cfg["bin_centers"] = model.bin_centers.detach().cpu().tolist()
+    (path / "config.json").write_text(json.dumps(cfg, indent=2))

experiments/region-coherence-20260928/source/objectives.py

@@ -0,0 +1,17 @@
+import math
+import torch
+import torch.nn.functional as F
+
+def color_loss(pred,target):
+    """Fit one coherent target; distribution matching discourages gray averages."""
+    pred=F.adaptive_avg_pool2d(pred,(64,64)).float();target=target.float()
+    pixel=F.l1_loss(pred/20,target/20)
+    p=F.avg_pool2d(pred,2).flatten(2);t=F.avg_pool2d(target,2).flatten(2)
+    angle=torch.arange(8,device=p.device,dtype=p.dtype)*math.pi/8
+    directions=torch.stack([angle.cos(),angle.sin()],1)
+    a=torch.einsum('kc,bcn->bkn',directions,p).sort(-1).values
+    b=torch.einsum('kc,bcn->bkn',directions,t).sort(-1).values
+    distribution=F.l1_loss(a/20,b/20)
+    # Match boundaries in the teacher target, never minimize gradients toward zero.
+    edge=(F.l1_loss((pred[:,:,:,1:]-pred[:,:,:,:-1])/20,(target[:,:,:,1:]-target[:,:,:,:-1])/20)+F.l1_loss((pred[:,:,1:]-pred[:,:,:-1])/20,(target[:,:,1:]-target[:,:,:-1])/20))/2
+    return pixel+.25*distribution+.1*edge,{'pixel':pixel,'distribution':distribution,'edge':edge}

experiments/region-coherence-20260928/source/persistence.py

@@ -0,0 +1,42 @@
+"""Commit run artifacts atomically and verify every changed file by SHA256."""
+import hashlib, json, os, time
+from pathlib import Path
+from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_download
+
+REPO = 'User-2468/mini-unet-colorizer'
+PREFIX = 'experiments/round6-20260928'
+
+class DurableRun:
+    def __init__(self, root):
+        if not os.environ.get('HF_TOKEN'):
+            raise RuntimeError('A write token is required before starting')
+        self.api = HfApi(token=os.environ['HF_TOKEN'])
+        if self.api.whoami()['name'] != 'User-2468':
+            raise RuntimeError('Unexpected account')
+        self.root = Path(root)
+        self.saved = {}
+
+    def sync(self, reason):
+        files = [p for p in sorted(self.root.rglob('*')) if p.is_file()]
+        hashes = {p.relative_to(self.root).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() for p in files}
+        changed = [p for p in files if self.saved.get(p.relative_to(self.root).as_posix()) != hashes[p.relative_to(self.root).as_posix()]]
+        if not changed:
+            return
+        for attempt in range(3):
+            try:
+                head = self.api.model_info(REPO, revision='main').sha
+                operations = [CommitOperationAdd(path_in_repo=PREFIX+'/'+p.relative_to(self.root).as_posix(), path_or_fileobj=str(p)) for p in changed]
+                operations.append(CommitOperationAdd(path_in_repo=PREFIX+'/SHA256SUMS.json', path_or_fileobj=json.dumps(hashes,sort_keys=True,indent=2).encode()))
+                commit = self.api.create_commit(repo_id=REPO, revision='main', parent_commit=head, operations=operations, commit_message='Colorizer round6: '+reason)
+                for p in changed:
+                    name = p.relative_to(self.root).as_posix()
+                    downloaded = hf_hub_download(REPO, PREFIX+'/'+name, revision=commit.oid, force_download=True, token=os.environ['HF_TOKEN'])
+                    if hashlib.sha256(Path(downloaded).read_bytes()).hexdigest() != hashes[name]:
+                        raise RuntimeError('Remote checksum mismatch: '+name)
+                self.saved = hashes
+                print('PERSISTED',reason,commit.oid,'verified_files',len(changed),flush=True)
+                return
+            except Exception:
+                if attempt == 2:
+                    raise
+                time.sleep(2**attempt)

experiments/region-coherence-20260928/source/semantic_model.py

@@ -0,0 +1,94 @@
+"""Compact pretrained semantic colorizer; dense and shared-palette variants."""
+import json
+from pathlib import Path
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torchvision.models import mobilenet_v3_large, MobileNet_V3_Large_Weights
+from safetensors.torch import save_file, load_file
+
+class QueryBlock(nn.Module):
+    def __init__(self,d=96):
+        super().__init__()
+        self.self_attn=nn.MultiheadAttention(d,4,batch_first=True,dropout=0)
+        self.cross_attn=nn.MultiheadAttention(d,4,batch_first=True,dropout=0)
+        self.norms=nn.ModuleList([nn.LayerNorm(d) for _ in range(3)])
+        self.ff=nn.Sequential(nn.Linear(d,2*d),nn.GELU(),nn.Linear(2*d,d))
+    def forward(self,q,memory):
+        x=self.norms[0](q);q=q+self.self_attn(x,x,x,need_weights=False)[0]
+        x=self.norms[1](q);q=q+self.cross_attn(x,memory,memory,need_weights=False)[0]
+        return q+self.ff(self.norms[2](q))
+
+def refine(d):
+    return nn.Sequential(nn.Conv2d(d,d,3,padding=1,bias=False),nn.GroupNorm(8,d),nn.SiLU())
+
+class SemanticColorizer(nn.Module):
+    def __init__(self,head='palette',pretrained=False,width=128,queries=16):
+        super().__init__()
+        if head not in ['palette','dense']:raise ValueError(head)
+        self.config={'architecture':'SemanticColorizer','head':head,'width':width,'queries':queries,'format_version':1}
+        self.encoder=mobilenet_v3_large(weights=MobileNet_V3_Large_Weights.IMAGENET1K_V2 if pretrained else None,progress=False).features
+        self.lateral=nn.ModuleList([nn.Conv2d(c,width,1) for c in [24,40,112,960]])
+        self.refine=nn.ModuleList([refine(width) for _ in range(3)])
+        self.register_buffer('rgb_mean',torch.tensor([.485,.456,.406]).view(1,3,1,1))
+        self.register_buffer('rgb_std',torch.tensor([.229,.224,.225]).view(1,3,1,1))
+        if head=='palette':
+            self.queries=nn.Parameter(torch.randn(queries,width)*.2)
+            self.query_blocks=nn.ModuleList([QueryBlock(width) for _ in range(2)])
+            self.memory_norm=nn.LayerNorm(width)
+            self.query_norm=nn.LayerNorm(width)
+            self.pixel=nn.Conv2d(width,width,1)
+            self.palette=nn.Sequential(nn.Linear(width,width),nn.GELU(),nn.Linear(width,2))
+            self.residual=nn.Conv2d(width,2,1)
+            nn.init.normal_(self.palette[-1].weight,std=.01);nn.init.zeros_(self.palette[-1].bias)
+            nn.init.zeros_(self.residual.weight);nn.init.zeros_(self.residual.bias)
+        else:
+            self.dense=nn.Sequential(refine(width),nn.Conv2d(width,2,1))
+            nn.init.normal_(self.dense[-1].weight,std=.01);nn.init.zeros_(self.dense[-1].bias)
+        count=sum(p.numel() for p in self.parameters())
+        if count>=4_000_000:raise ValueError(f'Parameter budget exceeded: {count}')
+
+    @staticmethod
+    def neutral_rgb(L):
+        light=(L.float()*50+50).clamp(0,100)
+        y=torch.where(light>8,((light+16)/116)**3,light/903.296296)
+        g=torch.where(y<=.0031308,12.92*y,1.055*y.clamp_min(1e-8).pow(1/2.4)-.055)
+        return g.expand(-1,3,-1,-1)
+
+    def forward(self,L):
+        h,w=L.shape[-2:]
+        x=F.pad(self.neutral_rgb(L),(0,(-w)%32,0,(-h)%32),mode='replicate')
+        x=(x-self.rgb_mean)/self.rgb_std
+        features=[]
+        for i,layer in enumerate(self.encoder):
+            x=layer(x)
+            if i in [3,6,12,16]:features.append(x)
+        projected=[layer(f) for layer,f in zip(self.lateral,features)]
+        x=projected[-1]
+        for i in range(2,-1,-1):
+            x=self.refine[2-i](F.interpolate(x,size=projected[i].shape[-2:],mode='bilinear',align_corners=False)+projected[i])
+        if self.config['head']=='palette':
+            memory=torch.cat([F.adaptive_avg_pool2d(f,(8,8)).flatten(2).transpose(1,2) for f in projected[1:]],1)
+            memory=self.memory_norm(memory)
+            q=self.queries[None].expand(L.shape[0],-1,-1)
+            for block in self.query_blocks:q=block(q,memory)
+            q=self.query_norm(q)
+            palette=80*torch.tanh(self.palette(q))
+            masks=torch.einsum('bqd,bdhw->bqhw',q,self.pixel(x))/(self.config['width']**.5)
+            weights=F.softmax(masks.float(),dim=1)
+            ab=torch.einsum('bqhw,bqc->bchw',weights,palette.float())+2*torch.tanh(self.residual(x).float())
+        else:ab=80*torch.tanh(self.dense(x).float())
+        return F.interpolate(ab,size=(x.shape[-2]*4,x.shape[-1]*4),mode='bilinear',align_corners=False)[...,:h,:w]
+    def decode(self,z,temperature=.38):return z.float()
+
+def save_semantic(model,path):
+    path=Path(path);path.mkdir(parents=True,exist_ok=True)
+    save_file({k:v.detach().cpu().contiguous() for k,v in model.state_dict().items()},str(path/'model.safetensors'))
+    (path/'config.json').write_text(json.dumps(model.config,indent=2))
+    (path/'README.md').write_text('# Experimental semantic colorizer\n\nHead: '+model.config['head']+'. Parameters: '+str(sum(p.numel() for p in model.parameters()))+'.\n\nNot approved for production. Requires semantic_model.py; incompatible with the old U-Net loader. Input is Lab lightness normalized to [-1,1]; output is Lab ab. See the run protocol, provenance, selection and visual comparisons. Predictions are plausible colors, not recovered historical truth.\n')
+
+def load_semantic(path,device='cpu'):
+    path=Path(path);cfg=json.loads((path/'config.json').read_text())
+    model=SemanticColorizer(**{k:cfg[k] for k in ['head','width','queries']})
+    model.load_state_dict(load_file(str(path/'model.safetensors')),strict=True)
+    return model.to(device).eval()

experiments/region-coherence-20260928/source/spatial.py

@@ -0,0 +1,34 @@
+"""Controlled spatial decoding alternatives; no learned parameters."""
+import math
+import torch
+import torch.nn.functional as F
+
+def box_mean(x,radius):
+    # Border-normalized local windows, also work on images smaller than kernel.
+    return F.avg_pool2d(x,2*radius+1,stride=1,padding=radius,count_include_pad=False)
+
+def guided_chroma(L,ab,radius=4,epsilon=.001):
+    """Scalar luminance-guided local linear filter (He et al., ECCV 2010).
+
+    L is the network's [-1,1] luminance. Epsilon is in [0,1] luminance units.
+    Uses luminance only; reference colors never enter inference.
+    """
+    if not isinstance(radius,int) or radius<0 or not math.isfinite(epsilon) or epsilon<=0:
+        raise ValueError('Invalid guided-filter radius or epsilon')
+    if radius==0:return ab
+    I=(L.float()+1)/2;p=ab.float()
+    mi=box_mean(I,radius);mp=box_mean(p,radius)
+    var=(box_mean(I*I,radius)-mi*mi).clamp_min(0)
+    cov=box_mean(I*p,radius)-mi*mp
+    a=cov/(var+epsilon);b=mp-a*mi
+    return box_mean(a,radius)*I+box_mean(b,radius)
+
+def spatial_decode(model,logits,L,temperature=.38,pool=1,radius=0,epsilon=.001):
+    if pool<1 or not isinstance(pool,int):raise ValueError('pool must be a positive integer')
+    if pool>1:
+        # Average evidence before annealing, then upsample chroma, not RGB.
+        small=F.avg_pool2d(logits,pool,ceil_mode=True,count_include_pad=False)
+        ab=model.decode(small,temperature)
+        ab=F.interpolate(ab,size=L.shape[-2:],mode='bilinear',align_corners=False)
+    else:ab=model.decode(logits,temperature)
+    return guided_chroma(L,ab,radius,epsilon)

experiments/region-coherence-20260928/train.py

@@ -0,0 +1,341 @@
+
+import os,sys,time,json,hashlib,shutil,random,copy,math,traceback,io
+from pathlib import Path
+import numpy as np
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch.utils.data import Dataset,DataLoader
+from torchvision.models.segmentation import deeplabv3_resnet50,DeepLabV3_ResNet50_Weights
+from skimage.segmentation import slic
+import cv2
+from PIL import ImageDraw
+from torchvision.transforms import RandomResizedCrop
+from torchvision.transforms import functional as TF
+from PIL import Image
+from skimage.color import rgb2lab
+import pyarrow as pa
+import pyarrow.parquet as pq
+from huggingface_hub import HfApi,hf_hub_download
+REPO='User-2468/mini-unet-colorizer'
+BASE='1a9eb8af2754ad2329a24cfe50d388cb559441d0'
+PREFIX='experiments/region-coherence-20260928'
+WORK=Path('/work');WORK.mkdir(exist_ok=True);os.chdir(WORK)
+START=time.monotonic();MAX_SECONDS=23*60
+api=HfApi(token=os.environ['HF_TOKEN']);assert api.whoami()['name']=='User-2468'
+src='experiments/final-20260928/realism_low_v2'
+source=WORK/'source';source.mkdir(exist_ok=True)
+sums=json.loads(Path(hf_hub_download(REPO,src+'/SHA256SUMS.json',revision=BASE)).read_text())
+for f in ['data.py','legacy_eval.py','metrics.py','model.py','objectives.py','persistence.py','previous_manifest.json','semantic_model.py','spatial.py']:
+    p=Path(hf_hub_download(REPO,src+'/source/'+f,revision=BASE))
+    assert hashlib.sha256(p.read_bytes()).hexdigest()==sums['source/'+f]
+    shutil.copy2(p,source/f)
+shutil.copy2(hf_hub_download(REPO,src+'/data_and_eval.py',revision=BASE),WORK/'r7.py')
+sys.path.insert(0,str(source));sys.path.insert(0,str(WORK))
+import r7 as r
+from semantic_model import load_semantic,save_semantic
+from data import decode_image
+r.persistence.PREFIX=PREFIX
+OUT=r.OUT;durable=r.persistence.DurableRun(OUT)
+initial=WORK/'initial';initial.mkdir(exist_ok=True)
+for f in ['model.safetensors','config.json']:
+    shutil.copy2(hf_hub_download(REPO,f,revision=BASE),initial/f)
+r.ev.deadline=lambda:None
+SEED=92851;STEPS=2000;BATCH=32
+HEAD=api.model_info(REPO,revision='main').sha
+CACHE_REV='7b10485ee9d205ce2f7b660dfc6a90012d78b1cb'
+torch.set_num_threads(8);torch.manual_seed(SEED);random.seed(SEED);np.random.seed(SEED)
+torch.backends.cudnn.benchmark=True
+torch.backends.cuda.matmul.allow_tf32=True
+torch.backends.cudnn.allow_tf32=True
+
+
+def coherence(pred,target,regions):
+    p=F.adaptive_avg_pool2d(pred.float(),(64,64))
+    t=target.float()
+    b=p.shape[0];idx=regions.long().reshape(b,1,-1)
+    err=(p-t).flatten(2);n=65
+    count=err.new_zeros(b,1,n).scatter_add_(2,idx,torch.ones_like(idx,dtype=err.dtype))
+    sums=err.new_zeros(b,2,n).scatter_add_(2,idx.expand(-1,2,-1),err)
+    mean=sums/count.clamp_min(1)
+    centered=(err-mean.gather(2,idx.expand(-1,2,-1))).abs().mean(1,keepdim=True)
+    rs=err.new_zeros(b,1,n).scatter_add_(2,idx,centered)/count.clamp_min(1)
+    valid=count>=16;valid[:,:,0]=False
+    region=(rs*valid).sum()/valid.sum().clamp_min(1)/20
+    # Penalize loss of chroma relative to the selected target, not universal saturation.
+    holes=(.65*t.norm(dim=1)-p.norm(dim=1)).relu()
+    mask=(t.norm(dim=1)>10)&(regions>0)
+    holes=(holes*mask).sum()/mask.sum().clamp_min(1)/20
+    return region,holes
+
+def objective(pred,target,regions,arm):
+    supervised=r.losses(pred,target).mean()
+    region,holes=coherence(pred,target,regions)
+    total=supervised+(.4*region+.15*holes if arm=='region' else 0)
+    return total,{'supervised':float(supervised.detach()),'region':float(region.detach()),'holes':float(holes.detach())}
+
+def smoke():
+    t=torch.zeros(2,2,64,64,device='cuda');t[:,0]=25
+    seg=torch.ones(2,64,64,device='cuda',dtype=torch.long)
+    # Whole-object hue shifts preserve internal structure; local defects do not.
+    assert coherence(t,t,seg)[0]==0
+    assert coherence(t+10,t,seg)[0]<1e-6
+    patch=t.clone();patch[:,:,20:40,20:40]=0
+    assert coherence(patch,t,seg)[0]>.01
+    assert coherence(torch.zeros_like(t),t,seg)[1]>.1
+    pattern=t.clone();pattern[:,:,:,::2]+=7
+    assert coherence(pattern,pattern,seg)[0]==0
+    p=patch.clone().requires_grad_(True);loss,_=objective(p,t,seg,'region')
+    loss.backward();assert torch.isfinite(p.grad).all() and p.grad.abs().sum()>0
+    m=load_semantic(initial,'cuda')
+    assert sum(v.numel() for v in m.parameters())==3994676
+    x=torch.zeros(2,1,256,256,device='cuda');loss,_=objective(m(x),t,seg,'region')
+    loss.backward()
+    assert all(torch.isfinite(v.grad).all() for v in m.parameters() if v.grad is not None)
+    save_semantic(m,OUT/'initial')
+    reload=load_semantic(OUT/'initial','cuda')
+    for k,v in m.state_dict().items():assert torch.equal(v,reload.state_dict()[k])
+    r.write('smoke.json',{'passed':True,'generator_parameters':3994676,'uniform_shift_allowed':True,'patch_penalized':True,'gray_collapse_penalized':True,'pattern_preserved':True,'finite_backward':True,'checkpoint_roundtrip':True})
+    del m,reload
+
+def make_regions(label,confidence,rgb,target):
+    # Foreground connected components are semantic regions; touching instances may merge.
+    output=np.zeros((64,64),np.uint8);candidates=[]
+    for cls in np.unique(label):
+        if cls==0:continue
+        n,cc,stats,_=cv2.connectedComponentsWithStats((label==cls).astype(np.uint8),8)
+        for j in range(1,n):
+            if stats[j,cv2.CC_STAT_AREA]<32:continue
+            mask=(cc==j)
+            if confidence[mask].mean()<.75:continue
+            inner=cv2.erode(mask.astype(np.uint8),np.ones((3,3),np.uint8)).astype(bool)&(confidence>.65)
+            if inner.sum()>=16:candidates.append((int(inner.sum()),inner,True))
+    # Homogeneous reference surfaces cover background categories outside VOC's 20 classes.
+    sp=slic(rgb,n_segments=80,compactness=10,start_label=1,channel_axis=-1)
+    for j in np.unique(sp):
+        mask=(sp==j)&(label==0)
+        if mask.sum()<16:continue
+        colors=target[:,mask]
+        if np.linalg.norm(colors-colors.mean(1,keepdims=True),axis=0).mean()>5:continue
+        candidates.append((int(mask.sum()),mask,False))
+    candidates.sort(key=lambda x:(x[2],x[0]),reverse=True)
+    semantic=0
+    for _,mask,is_semantic in candidates:
+        mask=mask&(output==0)
+        if mask.sum()<16:continue
+        nextid=int(output.max())+1
+        if nextid>64:break
+        output[mask]=nextid;semantic+=int(is_semantic)
+    return output,semantic
+
+class TrainPhotos(Dataset):
+    def __init__(self,rgb,targets,regions):
+        self.rgb,self.targets,self.regions=rgb,targets,regions
+    def __len__(self):return len(self.rgb)
+    def __getitem__(self,k):
+        rgb=self.rgb[k].astype(np.float32)/255
+        if random.random()<.5:
+            g=rgb@np.random.dirichlet([3.,5.,2.]).astype(np.float32)
+            g=np.clip((g**random.uniform(.8,1.25)-.5)*random.uniform(.75,1.15)+.5+random.uniform(-.05,.05),0,1)
+            if random.random()<.4:g=cv2.GaussianBlur(g,(5,5),random.uniform(.3,1.0))
+            if random.random()<.4:g=np.clip(g+np.random.normal(0,random.uniform(.002,.012),g.shape),0,1)
+        else:g=np.asarray(Image.fromarray(self.rgb[k]).convert('L'),np.float32)/255
+        L=rgb2lab(np.repeat(g[...,None],3,-1).astype(np.float32))[...,0][None]/50-1
+        target=self.targets[k].astype(np.float32);regions=self.regions[k].astype(np.int64)
+        if random.random()<.5:L=L[...,::-1];target=target[...,::-1];regions=regions[...,::-1]
+        return torch.from_numpy(L.copy()),torch.from_numpy(target.copy()),torch.from_numpy(regions.copy())
+
+def persist(models,opts,step,history):
+    for name,m in models.items():
+        save_semantic(m,OUT/name/f'step_{step:05d}')
+        torch.save({'optimizer':opts[name].state_dict(),'step':step,'torch_rng':torch.get_rng_state(),'cuda_rng':torch.cuda.get_rng_state_all(),'numpy_rng':np.random.get_state(),'python_rng':random.getstate(),'note':'Loader cursor not stored; data order after resume is not bit-exact'},OUT/name/'latest_optimizer.pt')
+    r.write('history.json',history)
+    r.write('status.json',{'status':'training','step':step,'elapsed_seconds':time.monotonic()-START,'production_approved':False})
+    durable.sync('region pilot step '+str(step))
+
+@torch.inference_mode()
+def fixed_losses(models,table,ids,segmenter,preprocess,name):
+    rows=[]
+    for i in ids:
+        im=decode_image(table,i).convert('RGB').resize((256,256))
+        rgb=np.asarray(im,np.float32)/255
+        L,ab,_=r.ev.gray_arrays(im)
+        x=torch.from_numpy(L)[None].cuda()
+        t=F.avg_pool2d(torch.from_numpy(ab)[None].cuda(),4)
+        logits=segmenter(preprocess(torch.from_numpy(rgb).permute(2,0,1)[None].cuda()))['out'].float()
+        prob=F.interpolate(logits,(64,64),mode='bilinear',align_corners=False).softmax(1)
+        conf,label=prob.max(1)
+        seg,_=make_regions(label[0].cpu().numpy(),conf[0].cpu().numpy(),cv2.resize(rgb,(64,64)),t[0].cpu().numpy())
+        rr=torch.from_numpy(seg.astype(np.int64))[None].cuda()
+        row={'id':int(i)}
+        for key,m in models.items():
+            _,terms=objective(m.eval()(x),t,rr,'region')
+            row[key]=terms
+        rows.append(row)
+    r.write(name+'.json',{'rows':rows,'summary':{key:{term:float(np.mean([row[key][term] for row in rows])) for term in ['supervised','region','holes']} for key in models},'warning':'Fixed originals and frozen masks permit global object hue offsets for region metric. Not a proof of perceptual quality. No moving critic.'})
+
+def main():
+    shutil.copytree(source,OUT/'source',dirs_exist_ok=True,ignore=shutil.ignore_patterns('__pycache__'))
+    shutil.copy2('/work/train.py',OUT/'train.py');shutil.copy2('/work/r7.py',OUT/'data_and_eval.py')
+    protocol={'base_revision':BASE,'artifact_revision':HEAD,'teacher_cache_revision':CACHE_REV,'seed':SEED,'generator_parameters':3994676,'steps_per_arm':STEPS,'batch_size':BATCH,
+      'arms':['control','region'],'matched':'Same fixed selected targets, input augmentation batches, initial weights, optimizer and updates. Treatment adds .4 object residual consistency + .15 local missing chroma.',
+      'supervised':'L1/20 + .25 sliced chroma distribution + .1 gradient difference (existing r.losses)',
+      'target_choice':'One complete original or cached DDColor target per training image, selected once by v3 complete-image loss. No per-pixel target mixing. Reject teacher proposal if it removes over40% of original chroma or has >1.5x+1.5 mean chroma edge strength.',
+      'region_loss':'Mean absolute spatial residual after subtracting regional mean prediction-target error. Allows constant alternative hue shifts while retaining target patterns.',
+      'segmentation':'Frozen torchvision DeepLabV3 ResNet50 COCO_WITH_VOC_LABELS_V1 on original RGB, default520px preprocessing. Confident connected foreground interiors + homogeneous original-color SLIC background surfaces. Max64 regions/image.',
+      'inference':'Unchanged architecture and one-pass inference; segmentation/teacher are training-only.',
+      'data':'Seeded8192 subset of original training fold; pinned Imagenette/COCO shards and verified cached DDColor. 50% mild archival-style input degradations.',
+      'evaluation':'Reused dense-run development and test are explicitly regression sets. New128 COCO val/0001 examples exclude round7 and dense-run evaluation IDs and training byte duplicates. New holdout used only after training; familiar archived-photo probes are not independent historical validation.',
+      'limitations':['20 foreground semantic classes only','touching same-class objects can merge','RGB-derived masks are imperfect','DDColor targets are fallible','teacher/encoder upstream pretraining overlap unknown','one seed and short pilot','synthetic film variation is not true archival training','frozen target selection can retain v3 biases'],
+      'budget':{'hard_timeout_minutes':30,'training_stop_minutes':23,'maximum_compute_usd':1.25},
+      'release_policy':'All checkpoints retained for review; no automatic production replacement'}
+    r.write('protocol.json',protocol);smoke();durable.sync('smoke and initial model verified')
+    print('PREFLIGHT_PASSED',flush=True)
+    tables=[]
+    for repo,rev,files in [('johnowhitaker/imagenette2-320','771c1310a2487e8076ede6b7d6307244aa8400af',['default/train/0000.parquet']),('detection-datasets/coco','26ddc382fe75dfc2a0655b5977e296ea10efebce',['default/train/0000.parquet','default/train/0001.parquet'])]:
+        for f in files:tables.append(pq.read_table(hf_hub_download(repo,f,repo_type='dataset',revision=rev),columns=['image']))
+    table=pa.concat_tables(tables);oldmanifest=json.loads((source/'previous_manifest.json').read_text())
+    allids=oldmanifest['train'];cache=np.empty((len(allids),2,64,64),np.float16)
+    cachesums=json.loads(Path(hf_hub_download(REPO,'experiments/round6-20260928/SHA256SUMS.json',revision=CACHE_REV)).read_text())
+    for start in range(0,len(allids),2048):
+        f=f'teacher_cache/{start:05d}.npz'
+        path=Path(hf_hub_download(REPO,'experiments/round6-20260928/'+f,revision=CACHE_REV))
+        assert hashlib.sha256(path.read_bytes()).hexdigest()==cachesums[f]
+        with np.load(path) as z:
+            assert z['indices'].tolist()==allids[start:start+len(z['indices'])]
+            cache[start:start+len(z['target'])]=z['target']
+    fresh=pq.read_table(hf_hub_download('detection-datasets/coco','default/val/0001.parquet',repo_type='dataset',revision='26ddc382fe75dfc2a0655b5977e296ea10efebce'),columns=['image'])
+    regression=json.loads(Path(hf_hub_download(REPO,'experiments/dense-critic-20260928/manifest.json',revision=HEAD)).read_text())
+    oldeval=json.loads(Path(hf_hub_download(REPO,'experiments/round7-20260928/manifest.json',revision=BASE)).read_text())
+    trainhash={hashlib.sha256(table['image'][i].as_py()['bytes']).digest() for i in allids}
+    excluded=set(regression['development']+regression['heldout']+oldeval['development']+oldeval['heldout'])
+    untouched=[int(i) for i in np.random.default_rng(SEED).permutation(len(fresh)) if int(i) not in excluded and hashlib.sha256(fresh['image'][int(i)].as_py()['bytes']).digest() not in trainhash][:128]
+    assert len(untouched)==128
+    dev=regression['development'];held=regression['heldout']
+    evalhash={hashlib.sha256(fresh['image'][i].as_py()['bytes']).digest() for i in dev+held+untouched}
+    positions=[int(k) for k in np.random.default_rng(SEED).permutation(len(allids)) if hashlib.sha256(table['image'][allids[int(k)]].as_py()['bytes']).digest() not in evalhash][:8192]
+    ids=[allids[k] for k in positions];cache=cache[positions]
+    r.write('manifest.json',{'train':ids,'original_train_positions':positions,'development_regression':dev,'heldout_regression':held,'new_holdout':untouched,'coco_revision':'26ddc382fe75dfc2a0655b5977e296ea10efebce','imagenette_revision':'771c1310a2487e8076ede6b7d6307244aa8400af','exact_duplicate_filter':True,'upstream_pretraining_overlap_unknown':True,'near_duplicates_not_excluded':True})
+    baseline=load_semantic(initial,'cuda').eval().requires_grad_(False)
+    weights=DeepLabV3_ResNet50_Weights.COCO_WITH_VOC_LABELS_V1
+    segmenter=deeplabv3_resnet50(weights=weights,progress=False).cuda().eval().requires_grad_(False)
+    preprocess=weights.transforms()
+    r.write('segmentation_teacher.json',{'weights':str(weights),'url':weights.url,'parameters':sum(p.numel() for p in segmenter.parameters()),'categories':weights.meta['categories'],'preprocess':str(preprocess)})
+    rgbcache=np.empty((len(ids),256,256,3),np.uint8)
+    targets=np.empty_like(cache);regions=np.empty((len(ids),64,64),np.uint8)
+    teacher_choices=[];region_counts=[];foreground_counts=[];preview=Image.new('RGB',(256*4,280*8),'white')
+    for start in range(0,len(ids),16):
+        if time.monotonic()-START>16*60:raise TimeoutError('Preparation exceeded16min; aborting before training to save remaining budget')
+        chosen=ids[start:start+16];ls=[];original=[]
+        for j,i in enumerate(chosen):
+            im=decode_image(table,i).convert('RGB').resize((256,256),Image.Resampling.BILINEAR)
+            rgbcache[start+j]=np.asarray(im)
+            L,ab,_=r.ev.gray_arrays(im);ls.append(L);original.append(ab)
+        x=torch.from_numpy(np.stack(ls)).cuda()
+        o=F.avg_pool2d(torch.from_numpy(np.stack(original)).cuda(),4)
+        t=torch.from_numpy(cache[start:start+len(chosen)].astype(np.float32)).cuda()
+        rgb=torch.from_numpy(rgbcache[start:start+len(chosen)]).cuda().permute(0,3,1,2).float()/255
+        with torch.inference_mode(),torch.autocast('cuda',dtype=torch.bfloat16):
+            bp=baseline(x)
+            logits=segmenter(preprocess(rgb))['out']
+        cp=F.interpolate(logits.float(),(64,64),mode='bilinear',align_corners=False).softmax(1)
+        conf,label=cp.max(1)
+        with torch.no_grad():
+            colorok=(t.norm(dim=1).mean((1,2))>=.6*o.norm(dim=1).mean((1,2)))
+            edge=lambda z:torch.diff(z,dim=-1).abs().mean((1,2,3))+torch.diff(z,dim=-2).abs().mean((1,2,3))
+            goodedge=edge(t)<=1.5*edge(o)+1.5
+            use=(r.losses(bp,t)<r.losses(bp,o))&colorok&goodedge
+            target=torch.where(use[:,None,None,None],t,o)
+        targetnp=target.cpu().numpy()
+        for j in range(len(chosen)):
+            k=start+j
+            reg,nfg=make_regions(label[j].cpu().numpy(),conf[j].cpu().numpy(),rgbcache[k,::4,::4].astype(np.float32)/255,o[j].cpu().numpy())
+            regions[k]=reg;targets[k]=targetnp[j].astype(np.float16)
+            teacher_choices.append(bool(use[j]));region_counts.append(int(reg.max()));foreground_counts.append(nfg)
+            if k<8:
+                preview.paste(Image.fromarray(rgbcache[k]),(0,k*280+24))
+                colors=np.random.default_rng(123).integers(20,255,(65,3),dtype=np.uint8);colors[0]=0
+                preview.paste(Image.fromarray(colors[reg]).resize((256,256),Image.Resampling.NEAREST),(256,k*280+24))
+                full=F.interpolate(target[j:j+1].float(),(256,256),mode='bilinear',align_corners=False)[0].cpu().numpy().transpose(1,2,0)
+                preview.paste(r.toimage(ls[j],full),(512,k*280+24))
+                fullbp=bp[j].float().cpu().numpy().transpose(1,2,0)
+                preview.paste(r.toimage(ls[j],fullbp),(768,k*280+24))
+                ImageDraw.Draw(preview).text((2,k*280),f'id {chosen[j]} | RGB | region interiors | selected target | v3; DDColor={bool(use[j])}',fill='black')
+        if start%2048==0:print('PREP',start,'of',len(ids),flush=True)
+    if np.mean(np.array(region_counts)>0)<.5:raise RuntimeError('Insufficient usable region coverage')
+    preview.save(OUT/'training_targets_and_regions.jpg',quality=92)
+    r.write('target_selection.json',{'teacher_choice_fraction':float(np.mean(teacher_choices)),'teacher_choices':teacher_choices,'mean_regions':float(np.mean(region_counts)),'mean_semantic_regions':float(np.mean(foreground_counts)),'fraction_with_semantic_region':float(np.mean(np.array(foreground_counts)>0)),'region_counts':region_counts})
+    for start in range(0,len(ids),1024):
+        folder=OUT/'region_cache';folder.mkdir(exist_ok=True)
+        np.savez_compressed(folder/f'{start:05d}.npz',indices=np.array(ids[start:start+1024]),target=targets[start:start+1024],regions=regions[start:start+1024])
+    r.write('baseline_development.json',r.ev.evaluate(baseline,fresh,dev))
+    durable.sync('fixed targets and semantic regions verified')
+    models={name:load_semantic(initial,'cuda') for name in ['control','region']}
+    opts={name:torch.optim.AdamW([{'params':m.encoder.parameters(),'lr':3e-6,'base_lr':3e-6},{'params':[p for k,p in m.named_parameters() if not k.startswith('encoder.')],'lr':3e-5,'base_lr':3e-5}],weight_decay=.01) for name,m in models.items()}
+    ds=TrainPhotos(rgbcache,targets,regions)
+    loader=DataLoader(ds,batch_size=BATCH,shuffle=True,num_workers=6,pin_memory=True,drop_last=True,persistent_workers=True,generator=torch.Generator().manual_seed(SEED))
+    it=iter(loader);history=[];step=0
+    with torch.no_grad():
+        fixed_ids=np.arange(min(32,len(ids)))
+        fixedL=torch.from_numpy(np.stack([r.ev.gray_arrays(Image.fromarray(rgbcache[i]))[0] for i in fixed_ids])).cuda()
+        fixedT=torch.from_numpy(targets[fixed_ids].astype(np.float32)).cuda()
+        fixedR=torch.from_numpy(regions[fixed_ids].astype(np.int64)).cuda()
+    with torch.inference_mode():
+        startp=baseline(fixedL)
+        for name in models:
+            sl,st=objective(startp,fixedT,fixedR,name)
+            history.append({'arm':name,'step':0,'fixed_training_probe_total':float(sl),'fixed_training_probe_components':st})
+    for update in range(1,STEPS+1):
+        if time.monotonic()-START>MAX_SECONDS:break
+        try:L,t,rr=next(it)
+        except StopIteration:it=iter(loader);L,t,rr=next(it)
+        L,t,rr=L.cuda(non_blocking=True),t.cuda(non_blocking=True),rr.cuda(non_blocking=True)
+        for name,m in models.items():
+            m.train()
+            for mod in m.encoder.modules():
+                if isinstance(mod,nn.BatchNorm2d):mod.eval()
+            opt=opts[name];opt.zero_grad(set_to_none=True)
+            mult=min(update/100,1)*(.2+.8*(1+math.cos(math.pi*update/STEPS))/2)
+            for group in opt.param_groups:group['lr']=group['base_lr']*mult
+            with torch.autocast('cuda',dtype=torch.bfloat16):p=m(L)
+            loss,terms=objective(p,t,rr,name)
+            loss.backward();torch.nn.utils.clip_grad_norm_(m.parameters(),1,error_if_nonfinite=True);opt.step()
+            if update%100==0:
+                with torch.inference_mode():
+                    fp=m.eval()(fixedL);fl,ft=objective(fp,fixedT,fixedR,name)
+                history.append({'arm':name,'step':update,'total':float(loss.detach()),'components':terms,'fixed_training_probe_total':float(fl),'fixed_training_probe_components':ft,'elapsed_seconds':time.monotonic()-START})
+                print('TRAIN',name,update,round(float(loss.detach()),4),'fixed',round(float(fl),4),flush=True)
+        step=update
+        if step%500==0:
+            persist(models,opts,step,history)
+            for name,m in models.items():
+                r.write(f'{name}/development_{step:05d}.json',r.ev.evaluate(m.eval(),fresh,dev))
+            r.grid({'v3':baseline,**models},fresh,dev[:24],f'development_{step:05d}')
+            durable.sync('development comparison '+str(step))
+    persist(models,opts,step,history)
+    for name,m in {'v3':baseline,**models}.items():
+        r.write(name+'/regression_heldout.json',r.ev.evaluate(m.eval(),fresh,held))
+        r.write(name+'/new_holdout.json',r.ev.evaluate(m.eval(),fresh,untouched))
+        r.review(m,fresh,untouched[:48],name+'_new_holdout')
+        r.ev.probes(m,name+'_probes')
+        if name!='v3':save_semantic(m,OUT/name/'candidate')
+    fixed_losses({'v3':baseline,**models},fresh,untouched[:32],segmenter,preprocess,'fixed_region_evaluation')
+    r.grid({'v3':baseline,**models},fresh,untouched[:32],'new_holdout_comparison')
+    r.grid({'v3':baseline,**models},fresh,held[:32],'regression_comparison')
+    import requests
+    for name,url in {'migrant_mother':'https://cdn.loc.gov/service/pnp/ppmsca/50200/50236v.jpg','power_house_mechanic':'https://www.archives.gov/exhibits/picturing_the_century/images/port_hine_022_v36.jpg'}.items():
+        try:
+            response=requests.get(url,timeout=15);response.raise_for_status()
+            im=Image.open(io.BytesIO(response.content));im.verify()
+            tb=pa.table({'image':[{'bytes':response.content,'path':None}]})
+            r.grid({'v3':baseline,**models},tb,[0],name)
+            r.write(name+'_source.json',{'url':url,'sha256':hashlib.sha256(response.content).hexdigest(),'use':'familiar historical development probe; original colors unknown'})
+        except Exception as e:r.write(name+'_error.json',{'error':str(e)})
+    r.write('status.json',{'status':'completed','steps_per_arm':step,'target_steps':STEPS,'stop_reason':'completed' if step==STEPS else 'time_budget','elapsed_seconds':time.monotonic()-START,'generator_parameters':3994676,'production_approved':False,'next':'Review control versus region versus v3; report all outcomes. This pilot does not prove plausibility or historical truth.'})
+    durable.sync('completed region ablation and comparisons')
+
+try:main()
+except BaseException:
+    r.write('failure.json',{'traceback':traceback.format_exc().replace(os.environ.get('HF_TOKEN','__EMPTY__'),'[REDACTED]')})
+    durable.sync('failure report and saved checkpoints')
+    raise