import {execFile} from 'node:child_process'; import {promisify} from 'node:util'; import {readFile,writeFile,mkdir} from 'node:fs/promises'; import {resolve,dirname} from 'node:path'; import {fileURLToPath} from 'node:url'; import {createHash} from 'node:crypto'; const exec=promisify(execFile); const AB=process.env.NANOJEV_AGENT_BROWSER_BIN??resolve(dirname(fileURLToPath(import.meta.url)),'../node_modules/.bin/agent-browser'); const ROOT=resolve(dirname(fileURLToPath(import.meta.url)),'..'); export const INSTRUCTION='Choose one allowed next browser action. Complete assigned inputs and interactions before advancing or submitting. When the current stage has no unfinished fields or interactions, choose its assigned advance action, then submit on the final review. The unfinished-input indicator does not describe availability of advance/submit actions. Use scroll to reveal an offscreen target. Do not change satisfied fields or use optional reset/cancel actions. While busy wait. DONE requires visible success. If blocked and no permitted action can help choose BLOCKED.'; export const SPECIALS={DONE:'Task is successfully saved; finish.',WAIT:'Page is busy; wait for a new observation.',BLOCKED:'Required progress is unavailable; stop for host assistance.'}; const INPUTS=new Set(['fill','type','clear','select','multiselect','check','uncheck','radio','upload']); const OPS=new Set([...INPUTS,'click','hover','dblclick','drag','focus','press','open','option','add','remove','link','tab']); const hash=value=>createHash('sha256').update(JSON.stringify(value)).digest('hex'); export function description(a){return `operation=${a.op}; purpose=${a.purpose}; current=${a.current}; expected=${a.expected}; viewport=${a.viewport}; enabled=${a.enabled}; blocked=${a.blocked}; target purpose=${a.sourcePurpose??a.purpose}.`;} export function stateText(s){return `Task: complete the assigned browser form and confirm successful saving.\nPage busy=${s.busy}; success=${s.success}; incomplete fields=${Boolean(s.fields)}; incomplete interactions=${Boolean(s.interactions)}; blocking dialog=${s.modal}; unfinished input/interaction in viewport=${Boolean(s.ready)}.`;} export class Browser{ constructor(session,{origins}={}){this.session=session;this.calls=0;this.ms=0;this.targetId=null;this.parentTargets=[];this.flags=['--session',session,'--pin-tab','--headed','false','--idle-timeout','0'];if(origins?.length)this.flags.push('--allowed-domains',[...new Set(origins.map(x=>new URL(x).hostname))].join(','));} async command(args){const start=performance.now();this.calls++;const {stdout}=await exec(AB,[...this.flags,'--json',...args],{timeout:20000,maxBuffer:8*1024*1024,encoding:'utf8'});this.ms+=performance.now()-start;const result=JSON.parse(stdout);if(!result.success)throw Error(`agent-browser: ${String(result.error).slice(0,200)}`);return result.data;} async observe(task){ const tabs=(await this.command(['tab'])).tabs.filter(x=>x.active);if(tabs.length!==1)throw Error('Ambiguous active tab'); if(this.targetId&&this.targetId!==tabs[0].targetId)throw Error('Active tab changed unexpectedly');this.targetId=tabs[0].targetId; const snapshot=await this.command(['snapshot']); if(!task.allowedOrigins.includes(new URL(snapshot.origin).origin))throw Error('Origin outside authorized task'); // Pure DOM read: no mutation, no access to fixture state, answers, receipts or request APIs. const expression=`JSON.stringify(((${readDom.toString()})(${JSON.stringify(task)})))`; const raw=await this.command(['eval',expression]);const dom=typeof raw.result==='string'?JSON.parse(raw.result):raw.result; const refs=Object.entries(snapshot.refs??{}).map(([ref,m])=>{const line=snapshot.snapshot.split('\n').find(l=>new RegExp('\\bref='+ref+'\\b').test(l))??'';return {ref,...m,disabled:/\[(?:[^\]]*,\s*)?disabled(?:,|\])/.test(line),checked:/\bchecked(?:=true)?(?:,|\])/.test(line)&&!line.includes('checked=false')};}); const actions=[];let fields=0,interactions=0; for(const c of dom.controls){ if(c.count>1)throw Error(`Ambiguous selector for ${c.label}`); if(!c.visible)continue; if(c.type==='password')throw Error('Secret input requires host helper'); const purpose=c.purpose??(INPUTS.has(c.op)?'input':'interaction'); if(!c.blocked&&c.expected!=='optional'&&c.current!=='match'){ if(purpose==='input')fields++; if(purpose==='interaction')interactions++; } // Disabled and modal-covered controls cannot be authorized executable actions. if(c.blocked||!c.enabled)continue; const role=c.role||refs.find(r=>r.name===c.label)?.role?.toLowerCase()||'button'; const a={...c,role,purpose,expected:c.expected??'assigned'}; actions.push(a); if(c.viewport!=='inside')actions.push({...a,op:'scroll',role:'viewport',purpose:'reveal',sourcePurpose:purpose}); } const ready=actions.filter(a=>a.op!=='scroll'&&a.current!=='match'&&a.expected==='assigned'&&a.viewport==='inside'&&['input','interaction','dismiss'].includes(a.purpose)).length; const s={busy:dom.busy,success:dom.success,fields,interactions,modal:dom.modal,ready}; const relevant={s,actions:actions.map(a=>({...a,value:undefined,condition:undefined,expect:undefined,when:undefined})),targetId:this.targetId}; return {state:stateText(s),s,actions,snapshot,dom,signature:hash(relevant)}; } async act(a){ if(!a.enabled||a.blocked)throw Error('Model selected a disabled or covered action'); if(a.op!=='scroll'&&a.viewport!=='inside')throw Error('Model attempted an offscreen action without scrolling'); if(a.href&&['click','link'].includes(a.op)&&!this.allowedOrigins.includes(new URL(a.href).origin))throw Error('Link destination outside task origins'); if(a.frame){ if(a.op==='scroll')await this.command(['scrollintoview',a.frame]); await this.command(['frame',a.frame]); try{return await this.act({...a,frame:null});}finally{await this.command(['frame','main']);} } const sel=a.selector; switch(a.op){ case 'fill': if(['date','time','datetime-local'].includes(a.type))return this.fillTemporal(a); return this.command(['fill',sel,String(a.value)]); case 'type': await this.command(['focus',sel]); await this.command(['press','End']); return this.command(['keyboard','type',String(a.value)]); case 'clear':return this.command(['fill',sel,'']); case 'select':return this.command(['select',sel,String(a.value)]); case 'multiselect':return this.command(['select',sel,...a.value.map(String)]); case 'check':case 'uncheck':case 'hover':case 'dblclick':case 'focus':return this.command([a.op,sel]); case 'scroll':return this.command(['scrollintoview',sel]); case 'press':return process.platform==='darwin'&&/^(Escape|Esc)$/i.test(a.key)?this.pressEscape():this.command(['press',a.key]); case 'drag':return this.command(['drag',sel,a.destination]); case 'upload':return this.command(['upload',sel,...(Array.isArray(a.value)?a.value:[a.value]).map(String)]); case 'tab':{ if(a.mode==='open'){ if(!this.allowedOrigins.includes(new URL(a.url).origin))throw Error('Tab URL outside task origins'); this.parentTargets.push(this.targetId);await this.command(['tab','new',a.url]); }else if(a.mode==='return'){ const parent=this.parentTargets.pop();if(!parent)throw Error('No authorized parent tab');await this.command(['tab',parent]); }else throw Error('Unsupported tab operation'); const tabs=(await this.command(['tab'])).tabs.filter(t=>t.active);if(tabs.length!==1)throw Error('Tab activation ambiguous');this.targetId=tabs[0].targetId;return; } case 'radio':case 'click':case 'open':case 'option':case 'add':case 'remove':case 'link':return this.command(['click',sel]); default:throw Error(`Unsupported runtime operation ${a.op}`); } } async pressEscape(){ // agent-browser 0.37.1 CLI press on this Mac emits a stream of unrelated // native Minus key events after Escape. Its documented input_keyboard // channel sends the intended native key once without that translation. const status=await this.command(['stream','status']); if(!status.enabled||!Number.isInteger(status.port)||status.port<1||status.port>65535)throw Error('agent-browser keyboard stream unavailable'); const ws=new WebSocket(`ws://127.0.0.1:${status.port}/?pacing=ack&maxFps=1`); try{ await new Promise((resolve,reject)=>{const timer=setTimeout(()=>reject(Error('Keyboard stream connection timed out')),5000);ws.onopen=()=>{clearTimeout(timer);resolve();};ws.onerror=()=>{clearTimeout(timer);reject(Error('Keyboard stream connection failed'));};}); for(const eventType of ['keyDown','keyUp'])ws.send(JSON.stringify({type:'input_keyboard',eventType,key:'Escape',code:'Escape',windowsVirtualKeyCode:27,modifiers:0})); this.streamKeyboardEvents=(this.streamKeyboardEvents??0)+2; await this.command(['wait','100']); }finally{ws.close();} } async fillTemporal(a){ const raw=String(a.value),parts=[]; if(a.type!=='time'){ const m=raw.match(/^(\d{4})-(\d{2})-(\d{2})/);if(!m)throw Error('Date must use ISO format'); parts.push([/День|\bDay\b/i,m[3]],[/Месяц|\bMonth\b/i,m[2]],[/Год|\bYear\b/i,m[1]]); } if(a.type!=='date'){ const m=raw.match(/(?:T|^)(\d{2}):(\d{2})(?::(\d{2}))?$/);if(!m)throw Error('Time must use HH:MM[:SS]'); parts.push([/Час|\bHour\b/i,m[1]],[/Минут|\bMinute\b/i,m[2]]);if(m[3])parts.push([/Секунд|\bSecond\b/i,m[3]]); } for(const [pattern,value] of parts){ const s=await this.command(['snapshot','-s',a.selector]); const found=Object.entries(s.refs).filter(([r,m])=>m.role==='spinbutton'&&pattern.test(m.name)&&new RegExp('\\bref='+r+'\\b').test(s.snapshot)); if(found.length!==1)throw Error('Date/time segment is missing or ambiguous in current locale'); await this.command(['click','@'+found[0][0]]); for(const digit of value)await this.command(['press',digit]); } await this.command(['press','Tab']); const result=await this.command(['get','value',a.selector]);if(result.value!==raw)throw Error('Date/time input did not preserve the assigned value');return result; } } function readDom(task){ const context=frame=>frame?document.querySelector(frame)?.contentDocument:document; const query=(s,frame)=>s?Array.from(context(frame)?.querySelectorAll(s)??[]):[]; const shown=e=>!!e&&!!e.getClientRects().length&&getComputedStyle(e).visibility!=='hidden'&&getComputedStyle(e).display!=='none'; const value=e=>!e?null:e.type==='checkbox'||e.type==='radio'?e.checked:e.tagName==='SELECT'&&e.multiple?Array.from(e.selectedOptions).map(o=>o.value):e.type==='file'?Array.from(e.files).map(f=>({name:f.name,size:f.size})):('value'in e?e.value:e.textContent.trim()); const same=(a,b)=>JSON.stringify(a)===JSON.stringify(b); const viewBounds=el=>{ const w=el.ownerDocument.defaultView,b={top:0,left:0,right:w.innerWidth,bottom:w.innerHeight}; for(let p=el.parentElement;p;p=p.parentElement){ const style=w.getComputedStyle(p),r=p.getBoundingClientRect(); if(/auto|scroll|hidden|clip/.test(style.overflowY)){b.top=Math.max(b.top,r.top+p.clientTop);b.bottom=Math.min(b.bottom,r.top+p.clientTop+p.clientHeight);} if(/auto|scroll|hidden|clip/.test(style.overflowX)){b.left=Math.max(b.left,r.left+p.clientLeft);b.right=Math.min(b.right,r.left+p.clientLeft+p.clientWidth);} }return b; }; const condition=(c,frame)=>{ if(!c)return false;frame=c.frame??frame;const els=query(c.selector,frame),e=els[0]; if(c.kind==='any')return c.conditions.some(x=>condition(x,frame)); if(c.kind==='all')return c.conditions.every(x=>condition(x,frame)); if(c.kind==='absent')return !e||!shown(e); if(c.kind==='visible')return shown(e); if(c.kind==='count')return els.filter(shown).length===c.value; if(c.kind==='text')return !!e&&e.textContent.trim()===c.value; if(c.kind==='includes')return !!e&&e.textContent.includes(c.value); if(c.kind==='attribute')return !!e&&e.getAttribute(c.attribute)===c.value; if(c.kind==='focused')return !!e&&e.ownerDocument.activeElement===e; if(c.kind==='property')return !!e&&e[c.property]===c.value; if(c.kind==='value')return same(value(e),c.value); return false; }; const openDialogs=query('dialog[open],[role="dialog"][aria-modal="true"]').filter(shown),dialog=openDialogs.at(-1); const controls=task.controls.map(c=>{ const els=query(c.selector,c.frame),e=els[0],frameElement=c.frame?document.querySelector(c.frame):null,visible=shown(e)&&(!c.frame||shown(frameElement))&&(!c.when||condition(c.when,c.frame)); if(!e)return {...c,count:0,visible:false}; const r=e.getBoundingClientRect(),bounds=viewBounds(e),fr=frameElement?.getBoundingClientRect(); let above=r.topbounds.bottom+1||r.right>bounds.right+1; if(frameElement){const outer=viewBounds(frameElement);above||=r.top+fr.topouter.bottom+1||r.right+fr.left>outer.right+1;} const viewport=above?'above':below?'below':'inside'; let matched=false; if(c.expect)matched=condition(c.expect,c.frame); else if(c.op==='check'||c.op==='radio')matched=e.checked===true; else if(c.op==='uncheck')matched=e.checked===false; else if(c.op==='clear')matched=value(e)===''; else if(c.op==='multiselect')matched=same([...value(e)].sort(),[...c.value].sort()); else if(c.op==='upload')matched=same(value(e).map(f=>f.name).sort(),(Array.isArray(c.value)?c.value:[c.value]).map(p=>p.split('/').at(-1)).sort()); else if(['fill','type','select'].includes(c.op))matched=same(value(e),c.finalValue??c.value); const current=matched?'match':value(e)===''||value(e)===false?'empty':'wrong'; const enabled=!e.disabled&&e.getAttribute('aria-disabled')!=='true'&&!e.closest('[inert]')&&(!['fill','type','clear'].includes(c.op)||!e.readOnly); const nativeRole=e.getAttribute('role')||(e.tagName==='SELECT'?(e.multiple?'listbox':'combobox'):e.tagName==='TEXTAREA'?'textbox':e.tagName==='INPUT'?({number:'spinbutton',checkbox:'checkbox',radio:'radio',file:'button'}[e.type]??'textbox'):e.tagName==='A'?'link':'button'); return {...c,role:c.role??nativeRole,count:els.length,visible,current,enabled,viewport,blocked:!!dialog&&!dialog.contains(frameElement??e),type:e.type??null,href:e.tagName==='A'?e.href:null,actual:value(e)}; }); return {controls,errors:query(task.errorSelector??'[role="alert"]').filter(shown).map(e=>e.textContent.trim()).filter(Boolean),busy:query(task.busySelector??'[aria-busy="true"]').some(shown),success:condition(task.success),modal:controls.some(c=>c.visible&&!c.blocked&&c.purpose==='dismiss')}; } export function validateTask(t){ if(!t.name||!t.url||!Array.isArray(t.controls)||!t.controls.length||!Array.isArray(t.allowedOrigins)||!t.allowedOrigins.includes(new URL(t.url).origin))throw Error('Invalid task contract'); if(t.controls.some(c=>!OPS.has(c.op)||!c.selector||!c.label))throw Error('Invalid control'); if(new Set(t.controls.map(c=>c.id)).size!==t.controls.length)throw Error('Duplicate control IDs'); return t; } export async function runTask(task,{session,endpoint,maxSteps=100,outDir=resolve(ROOT,'.local/runs'),setup=true}={}){ if(!endpoint)throw Error('An explicit NanoJev endpoint is required'); validateTask(task);session??=`nanojev-${task.name}-${Date.now()}`;const browser=new Browser(session,{origins:task.allowedOrigins});browser.allowedOrigins=task.allowedOrigins;await mkdir(outDir,{recursive:true}); if(setup){await browser.command(['open',task.url]);await browser.command(['set','viewport','1100','800']);} const runnerSha256=createHash('sha256').update(await readFile(fileURLToPath(import.meta.url))).digest('hex'); const start=performance.now(),trace=[],visits=new Map();let status='step_limit',error=null,last=null,repeats=0,modelIdentity=null; try{ for(let step=0;step600000){status='time_limit';break;} const observation=await browser.observe(task); observation.actions=observation.actions.filter(a=>(a.op==='scroll'||a.viewport==='inside')&&!(a.current==='match'&&a.expected==='assigned'&&['input','interaction','dismiss','reveal'].includes(a.purpose))); const criteria=Object.fromEntries(observation.actions.map((a,i)=>[`a${i}`,description(a)]));Object.assign(criteria,SPECIALS); const request={states:[{id:'browser',state:observation.state,questions:{next:{type:'choice',instructions:INSTRUCTION,criteria}}}]}; const then=performance.now();const response=await fetch(endpoint+'/api/evaluate',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(request),signal:AbortSignal.timeout(60000)});const result=await response.json(); if(!response.ok)throw Error(`NanoJev inference HTTP ${response.status}: ${result.error}`); const answer=result.states[0].answers.next;const identity=result.checkpoint.id; if(modelIdentity&&modelIdentity!==identity)throw Error('Model changed mid-run');modelIdentity=identity; const a=answer.choice.startsWith('a')?observation.actions[Number(answer.choice.slice(1))]:null; if(!a&&!Object.hasOwn(SPECIALS,answer.choice))throw Error('Invalid choice'); const row={step:step+1,request,answer,execution:result.execution,apiMs:performance.now()-then,action:a??answer.choice,before:observation.dom,executed:false};trace.push(row); const fresh=await browser.observe(task); if(fresh.signature!==observation.signature){row.skipped='stale_observation';continue;} if(answer.choice==='DONE'){status=fresh.s.success?'completed':'premature_done';break;} if(answer.choice==='BLOCKED'){status='model_blocked';break;} const key=hash([fresh.s,a?.op??answer.choice,a?.selector,a?.current,a?.viewport]);repeats=key===last?repeats+1:0;last=key; visits.set(key,(visits.get(key)??0)+1); if(answer.choice!=='WAIT'&&visits.get(key)>3){status='action_cycle';break;} if(answer.choice!=='WAIT'&&repeats>=2){status='no_progress';break;} if(answer.choice==='WAIT')await browser.command(['wait','--fn',`![...document.querySelectorAll(${JSON.stringify(task.busySelector??'[aria-busy="true"]')})].some(e=>e.getClientRects().length&&getComputedStyle(e).visibility!=='hidden')`]); else await browser.act(a); row.executed=true;console.log(JSON.stringify({run:task.name,step:step+1,choice:answer.choice,op:a?.op??'WAIT',label:a?.label,probability:answer.probabilities[answer.choice]})); } }catch(e){status='error';error=String(e.message);} const elapsedMs=performance.now()-start;let final=null,screenshot=null,evidenceError=null; try{final=(await browser.observe(task)).dom;screenshot=resolve(outDir,task.name+'.png');await browser.command(['screenshot','--full',screenshot]);} catch(e){evidenceError=String(e.message);screenshot=null;if(status==='completed'){status='error';error='Could not verify final browser state: '+evidenceError;}} const report={name:task.name,session,status,error,evidenceError,modelIdentity,runnerSha256,elapsedMs,browserCalls:browser.calls,browserMs:browser.ms,agentBrowserStreamKeyboardEvents:browser.streamKeyboardEvents??0,taskHash:hash(task),trace,final,screenshot}; await writeFile(resolve(outDir,task.name+'.json'),JSON.stringify(report,null,2));console.log(JSON.stringify({finished:task.name,status,error,steps:trace.length,elapsedMs}));return {report,browser}; }