assets/inference.worker-Gfj_BSLk.js

const e=`onnx-community/falcon-perception-onnx-webgpu`,t=`https://huggingface.co/${e}/resolve/main`,n={unified:{file:`unified_v2.onnx`,size:203e7},embed_tokens:{file:`embed_tokens.onnx`,size:256e6},coord_decoder:{file:`coord_decoder.onnx`,size:96e6},size_decoder:{file:`size_decoder.onnx`,size:96e6},anyup:{file:`anyup.onnx`,size:81e6},segm_head:{file:`segm_head.onnx`,size:9e6},img_projector:{file:`img_projector.onnx`,size:3e6},coord_encoder:{file:`coord_encoder.onnx`,size:2e6},size_encoder:{file:`size_encoder.onnx`,size:2e6}},r={dim:1024,n_layers:28,n_heads:16,head_dim:128,n_kv_heads:8,vocab_size:65536,max_seq_len:8192,segm_out_dim:256,coord_out_dim:2048,size_out_dim:2048,coord_token_id:240,size_token_id:241,seg_token_id:262,eos_id:11,img_id:227,image_cls_token_id:244,img_end_id:230,spatial_patch_size:16,temporal_patch_size:1,channel_size:3,rope_theta:1e4,presence_token_id:268,absence_token_id:269,end_of_query_token_id:263,start_of_query_token_id:264,ref_seg_token_id:258};function i(e,t=.6){if(e.length<=1)return e;let n=e.map((e,t)=>({det:e,idx:t,area:o(e.box)}));n.sort((e,t)=>t.area-e.area);let r=new Set,i=[];for(let{det:e,idx:o}of n){let n=!1;for(let r of i)if(a(e.box,r)>t){n=!0;break}n||(r.add(o),i.push(e.box))}return e.filter((e,t)=>r.has(t))}function a(e,t){let n=Math.max(e.x1,t.x1),r=Math.max(e.y1,t.y1),i=Math.min(e.x2,t.x2),a=Math.min(e.y2,t.y2),s=i-n,c=a-r;if(s<=0||c<=0)return 0;let l=s*c,u=o(e)+o(t)-l;return u>0?l/u:0}function o(e){return Math.max(0,e.x2-e.x1)*Math.max(0,e.y2-e.y1)}let s,c=null,l=!1,u=null;const d={};function f(e){self.postMessage(e)}function p(e){let t=d[e];if(!t)throw Error(`ONNX session not loaded: ${e}`);return t}async function m(e,n){let r=`${t}/${e}`,i=await fetch(r);if(!i.ok)throw Error(`Failed to fetch ${e}: ${i.status}`);let a=i.body?.getReader();if(!a)return i.arrayBuffer();let o=[],s=0;for(;;){let{done:t,value:r}=await a.read();if(t)break;o.push(r),s+=r.length,f({type:`progress`,progress:Math.round(s/n*100),loaded:s,total:n,file:e})}let c=new Uint8Array(s),l=0;for(let e of o)c.set(e,l),l+=e.length;return c.buffer}async function h(e,t,n,r){let i=`${t}.data`,[a,o]=await Promise.all([m(t,n*.01),m(i,n*.99)]),c=await s.InferenceSession.create(a,{executionProviders:r,externalData:[{path:i,data:new Uint8Array(o)}]});return console.log(`Loaded ${e}: inputs=[${c.inputNames}] outputs=[${c.outputNames}]`),c}async function g(){try{s=await import(`./ort.bundle.min-BZrO1_VY.js`);let t=typeof SharedArrayBuffer<`u`;s.env.wasm.numThreads=t?Math.min(4,navigator.hardwareConcurrency||2):1;let r=import(`./transformers.web-DaZ4EEMB.js`).then(async({AutoTokenizer:t})=>{try{c=await t.from_pretrained(e),console.log(`Tokenizer loaded`)}catch(e){console.warn(`AutoTokenizer failed, will use fallback:`,x(e,`unknown error`))}}),i=[`unified`,`embed_tokens`,`coord_decoder`,`size_decoder`,`segm_head`,`anyup`,`img_projector`,`coord_encoder`,`size_encoder`],a=0,o=Object.values(n).reduce((e,t)=>e+t.size,0),l=new Set([`unified`]);for(let e of i){let{file:t,size:r}=n[e];f({type:`progress`,progress:Math.round(a/o*100),loaded:a,total:o,file:t});let i=l.has(e)?[`wasm`]:[`webgpu`,`wasm`];try{d[e]=await h(e,t,r,i)}catch(n){console.warn(`${i[0]} failed for ${e}, trying WASM:`,x(n,`unknown error`));try{d[e]=await h(e,t,r,[`wasm`])}catch(t){console.error(`Failed to load ${e}:`,x(t,`unknown error`))}}a+=r}await r,c||console.log(`Using fallback tokenizer (hardcoded prompt tokens)`),f({type:`loaded`})}catch(e){f({type:`error`,message:x(e,`Failed to load models`)})}}function ee(e){let t=r.spatial_patch_size,n=e.width,i=e.height,a=Math.min(512/n,512/i,1),o=Math.max(t,Math.round(n*a/t)*t),s=Math.max(t,Math.round(i*a/t)*t),c=new OffscreenCanvas(o,s).getContext(`2d`);c.drawImage(e,0,0,o,s);let l=c.getImageData(0,0,o,s),u=new Float32Array(s*o*3),d=new Float32Array(3*s*o),f=s*o;for(let e=0;e<f;e++){let t=l.data[e*4+0]/255*2-1,n=l.data[e*4+1]/255*2-1,r=l.data[e*4+2]/255*2-1;u[e*3+0]=t,u[e*3+1]=n,u[e*3+2]=r,d[0*f+e]=t,d[1*f+e]=n,d[2*f+e]=r}return{pixels:u,pixelsChw:d,width:o,height:s,patchH:s/t,patchW:o/t}}function te(e,t){let n=Math.sqrt(t/e),r=Math.sqrt(e/t),i=new Float32Array(e*t*2);for(let a=0;a<e;a++){let o=e>1?r*(2*a/(e-1)-1):0;for(let e=0;e<t;e++){let r=t>1?n*(2*e/(t-1)-1):0,s=a*t+e;i[s*2+0]=o,i[s*2+1]=r}}return i}function _(e){u?.image&&u.image!==e.image&&u.image.close(),u=e}function v(e){if(l){_(e),console.log(`[detect] queued latest frame while inference is running`);return}y(e)}async function y(e){l=!0;let{image:t,query:n,requestId:a,sourceTimestampMs:o,capturedAtMs:m}=e,h=(e,t)=>{f({type:`result`,detections:e,inferenceMs:t,query:n,requestId:a,sourceTimestampMs:o,capturedAtMs:m})};if(console.log(`[detect] query="${n}" image=${t.width}x${t.height}`),!d.unified||!d.embed_tokens){f({type:`error`,message:`Not ready. Sessions: ${Object.keys(d).join(`,`)}`}),l=!1;return}try{let e=performance.now(),a=r.dim,o=r.spatial_patch_size,l=o*o*r.channel_size,{pixels:u,pixelsChw:f,width:m,height:g,patchH:_,patchW:v}=ee(t),y=_*v;console.log(`[detect] image: ${m}x${g}px, ${_}x${v}=${y} patches`);let x=`Segment these expressions in the image:<|start_of_query|>${n}<|REF_SEG|>`,S;if(c){let e=c.encode(x,{add_special_tokens:!1});S=Array.from(e),console.log(`[detect] text tokens (${S.length}):`,S)}else S=[37462,1978,20528,821,790,6883,549,264,258],console.log(`[detect] using fallback tokens (no query text):`,S);let re=[245,246,247,248],C=1+re.length+y+1+S.length,w=new BigInt64Array(C),T=0;w[T++]=BigInt(r.image_cls_token_id);for(let e of re)w[T++]=BigInt(e);let E=T;for(let e=0;e<y;e++)w[T++]=BigInt(r.img_id);w[T++]=BigInt(r.img_end_id);for(let e of S)w[T++]=BigInt(e);console.log(`[detect] total seq_len=${C}, token IDs: [${Array.from(w.slice(0,5)).map(Number)}...${Array.from(w.slice(-5)).map(Number)}]`);let ie=await p(`embed_tokens`).run({token_ids:new s.Tensor(`int64`,w,[1,C])}),D=new Float32Array(ie.embeddings.data),O=new Float32Array(y*l);for(let e=0;e<_;e++)for(let t=0;t<v;t++){let n=e*v+t;for(let r=0;r<o;r++)for(let i=0;i<o;i++){let a=e*o+r,s=t*o+i,c=(a*m+s)*3,d=n*l+(r*o+i)*3;O[d+0]=u[c+0],O[d+1]=u[c+1],O[d+2]=u[c+2]}}let ae=(await p(`img_projector`).run({pixel_patches:new s.Tensor(`float32`,O,[y,l])})).projected.data;for(let e=0;e<y;e++){let t=E*a+e*a,n=e*a;for(let e=0;e<a;e++)D[t+e]=ae[n+e]}let oe=D.slice(0,5);console.log(`[detect] embedding[0][0:5]: [${Array.from(oe).map(e=>e.toFixed(3)).join(`, `)}]`);let se=D.slice(E*a,E*a+5);console.log(`[detect] embedding[imgStart][0:5]: [${Array.from(se).map(e=>e.toFixed(3)).join(`, `)}]`);let ce=D.slice((C-1)*a,(C-1)*a+5);console.log(`[detect] embedding[last][0:5]: [${Array.from(ce).map(e=>e.toFixed(3)).join(`, `)}]`),console.log(`[detect] running encoder (seq_len=${C})...`);let k=new Float32Array(C),A=E+y+1,le=new Set([245,246,247,248,r.img_id,r.img_end_id]),j=new Float32Array(C);for(let e=0;e<C;e++){let t=Number(w[e]);j[e]=le.has(t)&&e<A?0:1}let M=0;for(let e=0;e<C;e++)M+=j[e],k[e]=M-1;let N=r.head_dim/4,ue=r.rope_theta,P=new Float32Array(C*N*2);for(let e=0;e<C;e++){let t=k[e];for(let n=0;n<N;n++){let r=t/ue**(2*n/(N*2));P[(e*N+n)*2+0]=Math.cos(r),P[(e*N+n)*2+1]=Math.sin(r)}}let F=new Uint8Array(C*C);for(let e=0;e<C;e++)for(let t=0;t<=e;t++)F[e*C+t]=1;for(let e=0;e<A;e++)for(let t=0;t<A;t++)F[e*C+t]=1;let I=r.vocab_size,L=r.coord_out_dim/2,de=r.n_layers,fe=r.n_heads,R=r.head_dim,pe=new Float32Array(de*2*1*fe*0*R),z=new Float32Array(C*2);z.fill(NaN);let B=te(_,v);for(let e=0;e<y;e++){let t=(E+e)*2;z[t+0]=B[e*2+0],z[t+1]=B[e*2+1]}console.log(`[detect] prefill (seq_len=${C})...`);let V=await p(`unified`).run({token_embeddings:new s.Tensor(`float32`,D,[1,C,a]),freqs_cis:new s.Tensor(`float32`,P,[1,C,N,2]),pos_hw:new s.Tensor(`float32`,z,[1,C,2]),attention_mask:new s.Tensor(`bool`,F,[1,C,C]),past_kv:new s.Tensor(`float32`,pe,[de,2,1,fe,0,R]),past_len:new s.Tensor(`int64`,new BigInt64Array([0n]),[])}),H=V.logits.data,me=V.new_kv,U=C,he=(C-1)*I,W=H[he+r.presence_token_id],G=H[he+r.absence_token_id];console.log(`[detect] prefill done: presence=${W.toFixed(2)}, absence=${G.toFixed(2)}`);let K=[];if(W<=G&&W<0){console.log(`[detect] object "${n}" not found`),h(K,performance.now()-e);return}let ge=V.hidden_states.data,q=null,_e=d.anyup;if(_e&&_===16&&v===16&&m===256&&g===256){let e=new Float32Array(a*_*v),t=_*v;for(let n=0;n<y;n++){let r=(E+n)*a,i=Math.floor(n/v),o=n%v,s=i*v+o;for(let n=0;n<a;n++)e[n*t+s]=ge[r+n]}let n=performance.now();try{q=(await _e.run({images:new s.Tensor(`float32`,f,[1,3,g,m]),lr_tokens:new s.Tensor(`float32`,e,[1,a,_,v])})).hr_features,console.log(`[detect] anyup done in ${(performance.now()-n).toFixed(0)}ms, hr_features=${q.dims.join(`x`)}`)}catch(e){console.warn(`[detect] anyup failed; segmentation will fall back:`,e),q=null}}let J=new Float32Array(ge.subarray((C-1)*a,C*a)),Y=H,X=k[C-1]+1,Z=null,Q=null,ve=new Set([r.eos_id,r.end_of_query_token_id]);for(let t=0;t<510;t++){let i=Y.length,o=b(Y,i-I,I);if(ve.has(o)){console.log(`[detect] stop token ${o} (${ne(o)}) at step ${t}`);break}if(o===r.absence_token_id){console.log(`[detect] <absence> at step ${t}, stopping`);break}if(o===r.coord_token_id){let e=await p(`coord_decoder`).run({hidden:new s.Tensor(`float32`,J,[1,a])}),n=new Float32Array(e.logits.data),r=.01,i=0,o=0,c=0,l=0;for(let e=0;e<100&&(i=b(n,0,L),o=b(n,L,L),c=i/(L-1),l=o/(L-1),K.some(e=>{let t=(e.box.x1+e.box.x2)/2,n=(e.box.y1+e.box.y2)/2;return Math.abs(t-c)<r&&Math.abs(n-l)<r}));e++)n[i]=-1/0,n[L+o]=-1/0;Z=[c,l],console.log(`[detect] step ${t}: <coord> → (${c.toFixed(3)}, ${l.toFixed(3)})`)}else if(o===r.size_token_id){let e=(await p(`size_decoder`).run({hidden:new s.Tensor(`float32`,J,[1,a])})).logits.data,n=b(e,0,L),r=b(e,L,L),i=Math.log2(1/L),o=2**(n/(L-1)*-i+i),c=2**(r/(L-1)*-i+i);Q=[o,c],console.log(`[detect] step ${t}: <size> → (h=${o.toFixed(3)}, w=${c.toFixed(3)})`)}else if(o===r.seg_token_id){if(Z&&Q){let[r,i]=Z,[o,c]=Q,l;if(q&&d.segm_head)try{let e=await p(`segm_head`).run({seg_token_hidden:new s.Tensor(`float32`,J,[1,a]),hr_features:q}),t=e.masks.data,[,n,r]=e.masks.dims,i=new Uint8Array(n*r);for(let e=0;e<i.length;e++)i[e]=t[e]>0?255:0;l={data:i,width:r,height:n}}catch(e){console.warn(`[detect] segm_head failed for object #${K.length+1}:`,e)}if(K.push({label:n,score:1/(1+Math.exp(-(W-G))),box:{x1:Math.max(0,r-c/2),y1:Math.max(0,i-o/2),x2:Math.min(1,r+c/2),y2:Math.min(1,i+o/2)},mask:l}),console.log(`[detect] step ${t}: <seg> → object #${K.length} xy=(${r.toFixed(2)},${i.toFixed(2)}) hw=(${o.toFixed(2)},${c.toFixed(2)})${l?` +mask`:``}`),h([...K],performance.now()-e),K.length>=100){console.log(`[detect] reached max 100 objects`);break}Z=null,Q=null}}else console.log(`[detect] step ${t}: ${ne(o)} (${o})`);let c;if(o===r.coord_token_id&&Z){let e=await p(`coord_encoder`).run({coords:new s.Tensor(`float32`,new Float32Array(Z),[1,2])});c=new Float32Array(e.encoded.data)}else if(o===r.size_token_id&&Q){let e=await p(`size_encoder`).run({sizes:new s.Tensor(`float32`,new Float32Array(Q),[1,2])});c=new Float32Array(e.encoded.data)}else{let e=await p(`embed_tokens`).run({token_ids:new s.Tensor(`int64`,new BigInt64Array([BigInt(o)]),[1,1])});c=new Float32Array(e.embeddings.data)}let l=new Float32Array(N*2);for(let e=0;e<N;e++){let t=X/ue**(2*e/(N*2));l[e*2+0]=Math.cos(t),l[e*2+1]=Math.sin(t)}let u=new Uint8Array(U+1);u.fill(1);let f=new Float32Array([NaN,NaN]),m=await p(`unified`).run({token_embeddings:new s.Tensor(`float32`,c,[1,1,a]),freqs_cis:new s.Tensor(`float32`,l,[1,1,N,2]),pos_hw:new s.Tensor(`float32`,f,[1,1,2]),attention_mask:new s.Tensor(`bool`,u,[1,1,U+1]),past_kv:me,past_len:new s.Tensor(`int64`,new BigInt64Array([BigInt(U)]),[])});J=new Float32Array(m.hidden_states.data),Y=m.logits.data,me=m.new_kv,U+=1,X+=1}let ye=K.length,$=i(K,.6),be=performance.now()-e;console.log(`[detect] done in ${be.toFixed(0)}ms, ${$.length} detections (${ye} raw, ${ye-$.length} merged by bbox-IoU NMS)`),h($,be)}catch(e){console.error(`[detect] error:`,e),f({type:`error`,message:x(e,`Inference failed`)})}finally{t.close(),l=!1;let e=u;u=null,e&&v(e)}}function b(e,t,n){let r=0,i=-1/0;for(let a=0;a<n;a++)e[t+a]>i&&(i=e[t+a],r=a);return r}function x(e,t){return e instanceof Error?e.message:t}function ne(e){return{[r.coord_token_id]:`<coord>`,[r.size_token_id]:`<size>`,[r.seg_token_id]:`<seg>`,[r.eos_id]:`<eos>`,[r.presence_token_id]:`<presence>`,[r.absence_token_id]:`<absence>`,[r.end_of_query_token_id]:`<end_of_query>`}[e]??`token_${e}`}self.onmessage=async e=>{switch(e.data.type){case`load`:await g();break;case`detect`:v(e.data);break}};