""" Action Execution Engine for Playwright/CDP. Translates atomic actions and speculative batches into real browser events with natural human-like mouse kinematics, keystroke cadence, and coordinate grounding. """ from __future__ import annotations import asyncio import math import random from typing import Any, Dict, List, Optional, Tuple from playwright.async_api import Page, Locator from miniswardbower.core.schemas import ( ActionBatch, BrowserAction, BrowserActionType, InteractiveElement, PrunedAXTree, ) from miniswardbower.browser.speculative_tool_caller import SpeculativeToolCaller class ActionExecutor: """Executes atomic actions and speculative batches on the Playwright page.""" def __init__(self, action_timeout_ms: int = 5000, enable_natural_dynamics: bool = True): self.action_timeout_ms = action_timeout_ms self.enable_natural_dynamics = enable_natural_dynamics self._current_mouse_pos: Tuple[float, float] = (100.0, 100.0) self.speculator = SpeculativeToolCaller(enabled=True) def _resolve_locator(self, page: Page, target: str, tree: Optional[PrunedAXTree]) -> Locator: """Resolves an element ID (@e1) or raw CSS selector to a Playwright Locator.""" if target.startswith("@e") and tree is not None: for elem in tree.elements: if elem.id == target: return page.locator(elem.selector) raise ValueError(f"Target mark {target} not found in active AXTree") return page.locator(target).first def _find_element_in_tree(self, target: str, tree: Optional[PrunedAXTree]) -> Optional[InteractiveElement]: if target.startswith("@e") and tree is not None: for elem in tree.elements: if elem.id == target: return elem return None async def _move_mouse_natural(self, page: Page, target_x: float, target_y: float) -> None: """ Moves mouse along a cubic Bézier curve with ease-in/ease-out acceleration and subtle micro-jitter, mimicking human motor kinematics. """ x0, y0 = self._current_mouse_pos x3, y3 = target_x, target_y dist = math.hypot(x3 - x0, y3 - y0) if dist < 5: await page.mouse.move(x3, y3) self._current_mouse_pos = (x3, y3) return # Generate lateral displacement for cubic Bézier control points deviation = min(60.0, dist * 0.25) side = random.choice([-1.0, 1.0]) normal_x = -(y3 - y0) / (dist + 1e-6) normal_y = (x3 - x0) / (dist + 1e-6) # Control point 1 (around 25%-35% along the path) t1 = random.uniform(0.25, 0.35) dev1 = deviation * random.uniform(0.5, 1.0) * side x1 = x0 + (x3 - x0) * t1 + normal_x * dev1 y1 = y0 + (y3 - y0) * t1 + normal_y * dev1 # Control point 2 (around 65%-75% along the path) t2 = random.uniform(0.65, 0.75) dev2 = deviation * random.uniform(0.3, 0.8) * side x2 = x0 + (x3 - x0) * t2 + normal_x * dev2 y2 = y0 + (y3 - y0) * t2 + normal_y * dev2 # Step count based on distance (8 to 16 steps for snappier motor kinematics) steps = max(8, min(16, int(dist / 35))) duration = min(0.22, max(0.08, dist / 2200.0)) step_delay = duration / steps for i in range(1, steps + 1): raw_t = i / steps # Smoothstep ease-in / ease-out: S(t) = 3t^2 - 2t^3 t = raw_t * raw_t * (3.0 - 2.0 * raw_t) # Cubic Bézier formula bx = (1 - t)**3 * x0 + 3 * (1 - t)**2 * t * x1 + 3 * (1 - t) * t**2 * x2 + t**3 * x3 by = (1 - t)**3 * y0 + 3 * (1 - t)**2 * t * y1 + 3 * (1 - t) * t**2 * x2 + t**3 * y3 # Subtle micro-jitter (sub-pixel) jitter_x = random.uniform(-0.5, 0.5) if i < steps else 0.0 jitter_y = random.uniform(-0.5, 0.5) if i < steps else 0.0 await page.mouse.move(bx + jitter_x, by + jitter_y) await asyncio.sleep(step_delay) self._current_mouse_pos = (x3, y3) async def _is_reactive_stream_target(self, locator: Locator) -> bool: """ Determines whether an input target requires incremental keystroke emission (e.g., search typeaheads, live suggestion comboboxes) vs atomic fast-fill. """ try: return await locator.evaluate("""(el) => { if (!el) return false; const tag = (el.tagName || '').toLowerCase(); const type = (el.getAttribute('type') || '').toLowerCase(); const role = (el.getAttribute('role') || '').toLowerCase(); const ariaAuto = (el.getAttribute('aria-autocomplete') || '').toLowerCase(); const idClass = [ el.id || '', el.className || '', el.name || '', el.getAttribute('placeholder') || '' ].join(' ').toLowerCase(); if (role === 'combobox' || role === 'searchbox' || ariaAuto === 'list' || ariaAuto === 'both') { return true; } if (type === 'search') { return true; } if (/search|query|suggest|typeahead|autocomplete|filter|find/i.test(idClass)) { return true; } return false; }""") except Exception: return False async def _type_natural(self, locator: Locator, text: str) -> None: """ Simulates natural human typing cadence with Gaussian-distributed keystroke delays and micro-pauses at punctuation/spaces for interactive streams. """ await locator.focus() await locator.fill("") for char in text: await locator.press_sequentially(char, delay=0) delay = max(0.015, random.gauss(0.035, 0.010)) if char in " .,?!;:-_\n": delay += random.uniform(0.02, 0.05) await asyncio.sleep(delay) async def _visual_highlight(self, page: Page, target: Optional[str], desc: str) -> None: try: await page.evaluate("""([target, desc]) => { const hudAction = document.getElementById('__minisward_hud_action'); if (hudAction) { hudAction.innerHTML = `⚡ ${desc}`; } if (target) { const el = document.querySelector(`[data-minisward-mark="${target}"]`); if (el) { el.style.outline = '3px solid #10b981'; el.style.boxShadow = '0 0 15px rgba(16, 185, 129, 0.8)'; setTimeout(() => { el.style.outline = ''; el.style.boxShadow = ''; }, 500); } } }""", [target, desc]) except Exception: pass async def execute_action( self, page: Page, action: BrowserAction, tree: Optional[PrunedAXTree] = None, ) -> Dict[str, Any]: """Executes a single atomic browser action with natural dynamics and coordinate tracking.""" op = action.op details: Dict[str, Any] = { "op": op.value, "target": action.target, "instruction": action.instruction, } if action.target and action.target.startswith('@e'): elem = self._find_element_in_tree(action.target, tree) if elem is None: raise ValueError('Target mark requires its observation tree') loc = self._resolve_locator(page, action.target, tree) if await loc.count() != 1: raise ValueError('Stale or ambiguous target; observe again') current = await loc.evaluate("""el => ({ identity_valid: !el.hasAttribute('data-minisward-target') || window.__MINISWARD_IDENTITIES?.get(el) === el.getAttribute('data-minisward-target'), tag: el.tagName.toLowerCase(), text: (el.innerText || el.textContent || '').trim().replace(/\\s+/g, ' ').slice(0,80) || null, aria_label: el.getAttribute('aria-label'), name: el.getAttribute('name'), placeholder: el.getAttribute('placeholder'), href: el.href || null })""", timeout=self.action_timeout_ms) if not current.pop('identity_valid'): raise ValueError('Target node was replaced; observe again') if any(current[key] != getattr(elem, key) for key in current): raise ValueError('Target meaning changed; observe again') if not await loc.is_visible() or not await loc.is_enabled(): raise ValueError('Target is no longer actionable') # Visual feedback for headed mode action_label = f"{op.value}({action.target or ''} {action.text or action.key or ''})".strip() await self._visual_highlight(page, action.target, action_label) if op == BrowserActionType.GOTO: if not action.url: raise ValueError("URL must be provided for goto action") goto_url = action.url if goto_url.startswith("http://app.local") and page.url: from urllib.parse import urlsplit current_parts = urlsplit(page.url) if current_parts.netloc: origin = f"{current_parts.scheme}://{current_parts.netloc}" goto_url = goto_url.replace("http://app.local", origin, 1) await page.goto(goto_url, wait_until="domcontentloaded", timeout=self.action_timeout_ms) details["url"] = goto_url elif op == BrowserActionType.CLICK: target_coords: Optional[Tuple[float, float]] = None target_bbox: Optional[Tuple[float, float, float, float]] = None # sPTC: Check if speculative Future pre-resolved this target spec_res = await self.speculator.claim_speculation(action) if spec_res and "click_coords" in spec_res and spec_res["click_coords"][0] > 0: target_coords = spec_res["click_coords"] target_bbox = spec_res["bbox"] details["sptc_hit"] = True elif action.coords: target_coords = action.coords target_bbox = (action.coords[0] - 5, action.coords[1] - 5, 10, 10) elif action.target: elem_info = self._find_element_in_tree(action.target, tree) loc = self._resolve_locator(page, action.target, tree) try: await loc.scroll_into_view_if_needed(timeout=2000) except Exception: pass box = await loc.bounding_box(timeout=self.action_timeout_ms) if box: target_bbox = (box["x"], box["y"], box["width"], box["height"]) jx = random.uniform(-0.15, 0.15) * box["width"] if self.enable_natural_dynamics else 0.0 jy = random.uniform(-0.15, 0.15) * box["height"] if self.enable_natural_dynamics else 0.0 target_coords = (box["x"] + box["width"] / 2 + jx, box["y"] + box["height"] / 2 + jy) elif elem_info and elem_info.bbox[2] > 0: eb = elem_info.bbox target_bbox = (eb[0], eb[1], eb[2], eb[3]) target_coords = (eb[0] + eb[2] / 2, eb[1] + eb[3] / 2) if target_coords: if self.enable_natural_dynamics: await self._move_mouse_natural(page, target_coords[0], target_coords[1]) if action.target: loc = self._resolve_locator(page, action.target, tree) await loc.click(timeout=self.action_timeout_ms) else: await page.mouse.down() await asyncio.sleep(random.uniform(0.04, 0.08)) await page.mouse.up() else: if action.target: await self._resolve_locator(page, action.target, tree).click(timeout=self.action_timeout_ms) else: await page.mouse.click(target_coords[0], target_coords[1]) details["click_coords"] = target_coords details["bbox"] = target_bbox else: loc = self._resolve_locator(page, action.target, tree) await loc.click(timeout=self.action_timeout_ms) elif op == BrowserActionType.TYPE: if not action.target or action.text is None: raise ValueError("Target and text required for type") loc = self._resolve_locator(page, action.target, tree) box = await loc.bounding_box(timeout=self.action_timeout_ms) if box: details["bbox"] = (box["x"], box["y"], box["width"], box["height"]) # Hybrid Input Protocol (v3 Acceleration): # Check explicit action flag or auto-detect reactive stream fields (search/autocomplete) is_stream = action.stream_input if is_stream is None: is_stream = await self._is_reactive_stream_target(loc) if is_stream and self.enable_natural_dynamics: await self._type_natural(loc, action.text) details["input_mode"] = "stream_natural" else: # High-speed atomic fill with standard input/change dispatch await loc.fill(action.text, timeout=self.action_timeout_ms) details["input_mode"] = "atomic_fill" details["text"] = action.text elif op == BrowserActionType.PRESS: key = action.key or "Enter" await page.keyboard.press(key) details["key"] = key elif op == BrowserActionType.SELECT: if not action.target or not action.value: raise ValueError("Target and value required for select") loc = self._resolve_locator(page, action.target, tree) await loc.select_option(action.value, timeout=self.action_timeout_ms) details["value"] = action.value elif op == BrowserActionType.SCROLL: direction = action.direction or "down" if direction not in ('up', 'down'): raise ValueError('Only vertical scrolling is supported') details['scroll_before'] = await page.evaluate('window.scrollY') dy = 600 if direction == "down" else -600 if self.enable_natural_dynamics: steps = 3 for _ in range(steps): await page.mouse.wheel(0, dy / steps) await asyncio.sleep(0.015) else: await page.mouse.wheel(0, dy) # Event-driven scroll hydration: trigger scroll/resize for IntersectionObservers immediately try: await page.evaluate("""() => { window.dispatchEvent(new Event('scroll')); window.dispatchEvent(new Event('resize')); }""") except Exception: pass await asyncio.sleep(0.05) details['scroll_after'] = await page.evaluate('window.scrollY') details["direction"] = direction elif op == BrowserActionType.HOVER: if action.coords: await self._move_mouse_natural(page, action.coords[0], action.coords[1]) elif action.target: loc = self._resolve_locator(page, action.target, tree) box = await loc.bounding_box(timeout=self.action_timeout_ms) if box and self.enable_natural_dynamics: cx = box["x"] + box["width"] / 2 cy = box["y"] + box["height"] / 2 await self._move_mouse_natural(page, cx, cy) else: await loc.hover(timeout=self.action_timeout_ms) elif op == BrowserActionType.EXTRACT: extracted_data: Any = {} if action.extract_selector: extracted_data = await page.evaluate("""(sel) => { const elements = Array.from(document.querySelectorAll(sel)); return elements.map(el => ({ tag: el.tagName.toLowerCase(), text: (el.innerText || '').trim(), attributes: Object.fromEntries(Array.from(el.attributes).map(a => [a.name, a.value])) })); }""", action.extract_selector) elif action.target: loc = self._resolve_locator(page, action.target, tree) extracted_data = await loc.evaluate("""(el) => { const tag = el.tagName.toLowerCase(); if (tag === 'table') { const rows = Array.from(el.querySelectorAll('tr')).map(r => Array.from(r.querySelectorAll('th, td')).map(c => c.innerText.trim()) ); return { type: 'table', rows: rows }; } if (tag === 'ul' || tag === 'ol') { const items = Array.from(el.querySelectorAll('li')).map(li => li.innerText.trim()); return { type: 'list', items: items }; } return { tag: tag, text: (el.innerText || '').trim(), value: el.value || null, attributes: Object.fromEntries(Array.from(el.attributes).map(a => [a.name, a.value])) }; }""") else: extracted_data = await page.evaluate("""() => { const tables = Array.from(document.querySelectorAll('table')).map(t => ({ type: 'table', rows: Array.from(t.querySelectorAll('tr')).map(r => Array.from(r.querySelectorAll('th, td')).map(c => c.innerText.trim()) ) })); const cards = Array.from(document.querySelectorAll('.result-card, tr, article, .item, [data-card]')); if (cards.length > 0) { return { tables: tables, items: cards.map(c => ({ tag: c.tagName.toLowerCase(), text: (c.innerText || '').trim().replace(/\\s+/g, ' ').slice(0, 300) })).filter(c => c.text.length > 0) }; } return { title: document.title, tables: tables, body_text: (document.body.innerText || '').slice(0, 1500) }; }""") details["extracted_data"] = extracted_data elif op == BrowserActionType.WAIT: await asyncio.sleep(1.0) elif op == BrowserActionType.FINISH: details["finished"] = True return details async def execute_batch( self, page: Page, batch: ActionBatch, tree: Optional[PrunedAXTree] = None, ) -> List[Dict[str, Any]]: """ Executes a sequence of contiguous actions with speculative pipelining. Halts on first failure and returns execution trace for each action. """ results = [] for action in batch.actions: res = await self.execute_action(page, action, tree) results.append(res) pause = random.uniform(0.12, 0.22) if self.enable_natural_dynamics else 0.08 await asyncio.sleep(pause) return results async def execute_chunk( self, page: Page, actions: List[BrowserAction], tree: Optional[PrunedAXTree] = None, early_stop_on_navigation: bool = True, ) -> List[Dict[str, Any]]: """ Executes an atomic action chunk (AsyncWebRL 2026 / Action-Chunking). Pipelined execution across contiguous targets without intermediate re-scans. Halts early if navigation occurs or an action encounters an error. """ results = [] pre_url = page.url # sPTC (Speculative Programmatic Tool Calling, Aug 2026): # Pre-speculate element coordinates and bounding boxes for upcoming actions in background if len(actions) > 1 and tree is not None: def _resolve_elem_coords(target_id: str, ax_tree: PrunedAXTree): elem = self._find_element_in_tree(target_id, ax_tree) if elem and elem.bbox[2] > 0: eb = elem.bbox return (eb[0] + eb[2] / 2, eb[1] + eb[3] / 2, (eb[0], eb[1], eb[2], eb[3])) return (0.0, 0.0, (0.0, 0.0, 0.0, 0.0)) for upcoming in actions[1:]: if upcoming.target: self.speculator.speculate_action(upcoming, tree, _resolve_elem_coords) for idx, action in enumerate(actions): res = await self.execute_action(page, action, tree) results.append(res) # Check for URL navigation or state change if early_stop_on_navigation and idx < len(actions) - 1: try: if page.url != pre_url: res["early_stopped_due_to_navigation"] = True break except Exception: pass # Brisk human pipelining pause (15-30ms) between chunk actions pause = random.uniform(0.015, 0.035) if self.enable_natural_dynamics else 0.005 await asyncio.sleep(pause) return results