import os import mujoco import numpy as np STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") MESH_FILE = os.path.join(STATIC_DIR, "so101_meshes.bin") TEXTURED = {"cork": [0.80, 0.66, 0.48, 1.0]} SKIP_PREFIX = ("wall",) def export_meshes(model): os.makedirs(STATIC_DIR, exist_ok=True) manifest = [] chunks = [] offset = 0 for i in range(model.nmesh): va, vn = model.mesh_vertadr[i], model.mesh_vertnum[i] fa, fn = model.mesh_faceadr[i], model.mesh_facenum[i] verts = model.mesh_vert[va : va + vn].astype(np.float32).tobytes() faces = model.mesh_face[fa : fa + fn].astype(np.uint16).tobytes() pad = (-len(faces)) % 4 manifest.append( {"v_off": offset, "v_n": int(vn), "f_off": offset + len(verts), "f_n": int(fn)} ) chunks += [verts, faces, b"\0" * pad] offset += len(verts) + len(faces) + pad with open(MESH_FILE, "wb") as f: f.write(b"".join(chunks)) return manifest def _rgba(model, g): mat = model.geom_matid[g] if mat >= 0: name = model.material(mat).name if name in TEXTURED: return TEXTURED[name] return [round(float(x), 3) for x in model.mat_rgba[mat]] return [round(float(x), 3) for x in model.geom_rgba[g]] def scene_description(model): geoms = [] bodies = set() for g in range(model.ngeom): group = model.geom_group[g] gtype = model.geom_type[g] name = model.geom(g).name if name.startswith(SKIP_PREFIX): continue if gtype == mujoco.mjtGeom.mjGEOM_MESH and group == 2: kind = "mesh" elif gtype == mujoco.mjtGeom.mjGEOM_BOX and group == 0: kind = "box" else: continue body = int(model.geom_bodyid[g]) if body: bodies.add(body) geoms.append( { "kind": kind, "body": body, "mesh": int(model.geom_dataid[g]) if kind == "mesh" else -1, "size": [round(float(x), 5) for x in model.geom_size[g]], "pos": [round(float(x), 5) for x in model.geom_pos[g]], "quat": [round(float(x), 6) for x in model.geom_quat[g]], "rgba": _rgba(model, g), } ) return geoms, sorted(bodies) def body_poses(data, bodies): out = [] for b in bodies: out += [round(float(x), 5) for x in data.xpos[b]] out += [round(float(x), 5) for x in data.xquat[b]] return out def reset_message(run, model, data, manifest, mesh_url, label=""): geoms, bodies = scene_description(model) return { "kind": "reset", "run": run, "geoms": geoms, "bodies": bodies, "meshes": manifest, "mesh_url": mesh_url, "poses": [body_poses(data, bodies)], "label": label, } def frames_message(run, start, poses, labels, done=False): return {"kind": "frames", "run": run, "start": start, "poses": poses, "labels": labels, "done": done} HTML = """
loading 3D view
drag to orbit, scroll to zoom, right-drag to pan
""" CSS = """ .f3a-wrap { position: relative; width: 100%; height: 520px; border-radius: 8px; overflow: hidden; background: #d9dadd; } .f3a-view { position: absolute; inset: 0; } .f3a-view canvas { display: block; width: 100%; height: 100%; } .f3a-hud { position: absolute; left: 10px; top: 8px; right: 10px; pointer-events: none; font: 13px/1.4 ui-monospace, monospace; color: #111; } .f3a-label { background: rgba(255,255,255,0.8); padding: 2px 6px; border-radius: 4px; } .f3a-bar { position: absolute; left: 10px; bottom: 8px; display: flex; gap: 10px; align-items: center; font: 12px ui-sans-serif, sans-serif; color: #333; } .f3a-replay { padding: 3px 10px; border-radius: 4px; border: 1px solid #888; background: #fff; cursor: pointer; } """ JS = """ const view = element.querySelector('.f3a-view'); const labelEl = element.querySelector('.f3a-label'); const st = { ready: false, run: null, frames: [], labels: [], shown: -1, playT0: 0, done: false, msgs: 0, bodyGroups: {}, meshCache: {}, meshUrl: null }; element.f3aState = st; let THREE = null; let renderer, scene, camera, controls, root, dynamicRoot; function applyPose(flat) { if (!flat) return; const ids = st.bodies || []; for (let i = 0; i < ids.length; i++) { const g = st.bodyGroups[ids[i]]; if (!g) continue; const o = i * 7; g.position.set(flat[o], flat[o + 1], flat[o + 2]); g.quaternion.set(flat[o + 4], flat[o + 5], flat[o + 6], flat[o + 3]); } } async function loadMeshes(url, manifest) { if (st.meshUrl === url) return; const buf = await (await fetch(url)).arrayBuffer(); st.meshCache = {}; manifest.forEach((m, i) => { const pos = new Float32Array(buf, m.v_off, m.v_n * 3); const idx = new Uint16Array(buf, m.f_off, m.f_n * 3); const geo = new THREE.BufferGeometry(); geo.setAttribute('position', new THREE.BufferAttribute(pos, 3)); geo.setIndex(new THREE.BufferAttribute(idx, 1)); geo.computeVertexNormals(); st.meshCache[i] = geo; }); st.meshUrl = url; } async function buildScene(msg) { await loadMeshes(msg.mesh_url, msg.meshes); if (dynamicRoot) { root.remove(dynamicRoot); } dynamicRoot = new THREE.Group(); root.add(dynamicRoot); st.bodyGroups = {}; st.bodies = msg.bodies; for (const b of msg.bodies) { const g = new THREE.Group(); st.bodyGroups[b] = g; dynamicRoot.add(g); } for (const gd of msg.geoms) { let geo; if (gd.kind === 'mesh') geo = st.meshCache[gd.mesh]; else geo = new THREE.BoxGeometry(gd.size[0] * 2, gd.size[1] * 2, gd.size[2] * 2); if (!geo) continue; const c = gd.rgba; const mat = new THREE.MeshStandardMaterial({ color: new THREE.Color(c[0], c[1], c[2]), roughness: 0.75, metalness: 0.05 }); const mesh = new THREE.Mesh(geo, mat); mesh.position.set(gd.pos[0], gd.pos[1], gd.pos[2]); mesh.quaternion.set(gd.quat[1], gd.quat[2], gd.quat[3], gd.quat[0]); mesh.castShadow = true; mesh.receiveShadow = true; (gd.body ? st.bodyGroups[gd.body] : dynamicRoot).add(mesh); } applyPose(msg.poses[0]); } function handle(msg) { if (!msg || !st.ready) return; if (msg.kind === 'reset') { st.run = msg.run; st.frames = []; st.labels = []; st.shown = -1; st.playT0 = performance.now(); st.done = false; st.building = true; buildScene(msg).then(() => { st.building = false; labelEl.textContent = msg.label || 'ready'; }); return; } st.msgs += 1; if (msg.kind === 'frames' && msg.run === st.run) { msg.poses.forEach((p, i) => { st.frames[msg.start + i] = p; st.labels[msg.start + i] = msg.labels[i]; }); if (msg.done) st.done = true; } } function available() { let n = st.shown; while (st.frames[n + 1]) n += 1; return n; } function tick() { requestAnimationFrame(tick); if (!renderer) return; const now = performance.now(); if (!st.building) { const avail = available(); const due = Math.floor((now - st.playT0) * 0.03); if (due > avail) st.playT0 = now - (avail + 1) / 0.03; const target = Math.min(due, avail); if (target > st.shown) { applyPose(st.frames[target]); labelEl.textContent = st.labels[target] || ''; st.shown = target; } } controls.update(); renderer.render(scene, camera); } function resize() { if (!renderer) return; const w = view.clientWidth || 640, h = view.clientHeight || 480; renderer.setSize(w, h, false); camera.aspect = w / h; camera.updateProjectionMatrix(); } element.querySelector('.f3a-replay').addEventListener('click', () => { st.shown = -1; st.playT0 = performance.now(); }); (async () => { THREE = await import('https://esm.sh/three@0.160.0'); const { OrbitControls } = await import('https://esm.sh/three@0.160.0/examples/jsm/controls/OrbitControls.js'); renderer = new THREE.WebGLRenderer({ antialias: true }); renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); renderer.shadowMap.enabled = true; renderer.shadowMap.type = THREE.PCFSoftShadowMap; view.appendChild(renderer.domElement); scene = new THREE.Scene(); scene.background = new THREE.Color(0xd9dadd); camera = new THREE.PerspectiveCamera(45, 1, 0.01, 20); camera.position.set(0.62, 0.45, 0.42); controls = new OrbitControls(camera, renderer.domElement); controls.target.set(0.14, 0.05, -0.03); controls.enableDamping = true; scene.add(new THREE.HemisphereLight(0xffffff, 0x8a8a8a, 1.6)); const sun = new THREE.DirectionalLight(0xffffff, 1.6); sun.position.set(0.6, 1.4, 0.5); sun.castShadow = true; sun.shadow.mapSize.set(2048, 2048); const sc = sun.shadow.camera; sc.left = -0.6; sc.right = 0.6; sc.top = 0.6; sc.bottom = -0.6; sc.near = 0.1; sc.far = 4; sun.shadow.bias = -0.0005; scene.add(sun); root = new THREE.Group(); root.rotation.x = -Math.PI / 2; scene.add(root); resize(); new ResizeObserver(resize).observe(view); st.ready = true; handle(props.value); requestAnimationFrame(tick); })().catch((e) => { labelEl.textContent = '3D view failed: ' + e; }); watch('value', () => handle(props.value)); """