Uploading model files

ATTRIBUTION.md

@@ -0,0 +1,9 @@
+# Attribution
+
+This policy includes weights derived from `facebook/metaclip-b16-fullcc2.5b` by Meta Platforms, Inc.
+
+MetaCLIP model page: https://huggingface.co/facebook/metaclip-b16-fullcc2.5b
+
+MetaCLIP paper: https://arxiv.org/abs/2309.16671
+
+Licensed under Creative Commons Attribution-NonCommercial 4.0 International (CC BY-NC 4.0).

CC-BY-NC-4.0.txt

@@ -0,0 +1,408 @@
+Attribution-NonCommercial 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution-NonCommercial 4.0 International Public
+License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial 4.0 International Public License ("Public
+License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You
+such rights in consideration of benefits the Licensor receives from
+making the Licensed Material available under these terms and
+conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+  d. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  e. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  f. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  g. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  h. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  i. NonCommercial means not primarily intended for or directed towards
+     commercial advantage or monetary compensation. For purposes of
+     this Public License, the exchange of the Licensed Material for
+     other material subject to Copyright and Similar Rights by digital
+     file-sharing or similar means is NonCommercial provided there is
+     no payment of monetary compensation in connection with the
+     exchange.
+
+  j. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  k. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  l. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part, for NonCommercial purposes only; and
+
+            b. produce, reproduce, and Share Adapted Material for
+               NonCommercial purposes only.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties, including when
+          the Licensed Material is used other than for NonCommercial
+          purposes.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+       4. If You Share Adapted Material You produce, the Adapter's
+          License You apply must not prevent recipients of the Adapted
+          Material from complying with this Public License.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database for NonCommercial purposes
+     only;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.
+

README.md

@@ -1,10 +1,8 @@
-# DINOv3 temporal action-chunk robotics VLA
-
-Self-contained policy produced by `train_dinov3_action_chunk_vla.py`.
+# MetaCLIP temporal action-chunk robotics VLA
 
 
 Upload every file in this directory. The adapter loads `model.safetensors`
 strictly and performs no model-hub or network access.
 
-Built with DINOv3. Use and redistribution are governed by the included
-`LICENSE.md`.
+MetaCLIP is licensed under CC BY-NC 4.0. Commercial use is not permitted by
+that license. See `CC-BY-NC-4.0.txt` and `ATTRIBUTION.md`.

flock_robotics_adapter.py

@@ -1,9 +1,9 @@
-"""Transactional FLock adapter for a self-contained DINOv3 action-chunk policy.
+"""Transactional FLock adapter for a self-contained MetaCLIP action-chunk policy.
 
 The exporter copies this file to ``flock_robotics_adapter.py`` next to
-``dinov3_action_chunk_model.py``, ``vla_config.json``, and
+``metaclip_action_chunk_model.py``, ``vla_config.json``, and
 ``model.safetensors``. Runtime loading is deliberately local-only: the complete
-DINOv3 backbone and policy head must already be present in the safetensors file.
+MetaCLIP backbone and policy head must already be present in the safetensors file.
 
 The validator can retry the same ``policy.act(obs)`` request.  State is therefore
 copy-on-write and is committed only after a valid action has been produced.  An
@@ -294,7 +294,7 @@ def _temporal_ensemble(
     return result, tuple(retained)
 
 
-class DINOv3ActionChunkPolicy:
+class MetaCLIPActionChunkPolicy:
     """Stateful action-chunk policy with transactional validator semantics."""
 
     def __init__(
@@ -304,14 +304,14 @@ class DINOv3ActionChunkPolicy:
         device: torch.device,
         dtype: torch.dtype,
         *,
-        text_vector_fn: Callable[[str, int], np.ndarray],
+        tokenizer: Any,
         phase_vector_fn: Callable[[int, int], np.ndarray],
     ) -> None:
         self.model = model
         self.config = dict(config)
         self.device = torch.device(device)
         self.dtype = dtype
-        self.text_vector_fn = text_vector_fn
+        self.tokenizer = tokenizer
         self.phase_vector_fn = phase_vector_fn
 
         self.history = int(self.config.get("history", DEFAULT_HISTORY))
@@ -345,6 +345,7 @@ class DINOv3ActionChunkPolicy:
             raise ValueError("gripper_hysteresis must be finite and non-negative")
 
         self._state: _RuntimeState | None = None
+        self._text_cache: dict[str, torch.Tensor] = {}
         self.model.to(device=self.device, dtype=self.dtype)
         self.model.eval()
 
@@ -406,16 +407,26 @@ class DINOv3ActionChunkPolicy:
             base.phase_history if base else (), phase_tensor, self.history
         )
 
-        text = (
-            f"task {parsed.task} difficulty {parsed.difficulty} "
-            f"instruction {parsed.instruction}"
-        )
-        text_array = _fixed_feature_array(
-            self.text_vector_fn(text, self.text_dim), self.text_dim, "text"
-        )
-        text_features = torch.as_tensor(
-            text_array, device=self.device, dtype=self.dtype
-        ).unsqueeze(0)
+        text_key = parsed.instruction.strip()
+        pending_text_cache: torch.Tensor | None = None
+        cached_text = self._text_cache.get(text_key)
+        if cached_text is None:
+            encoded = self.tokenizer(
+                text_key,
+                padding="max_length",
+                truncation=True,
+                max_length=77,
+                return_tensors="pt",
+            )
+            text_features = self.model.encode_text(
+                encoded["input_ids"], encoded["attention_mask"]
+            )
+            text_features = _single_text_feature(
+                text_features, self.text_dim, self.device, self.dtype
+            )
+            pending_text_cache = text_features.detach().clone()
+        else:
+            text_features = cached_text
         task_id = self.task_to_id.get(parsed.task, len(self.task_to_id))
         difficulty_id = self.difficulty_to_id.get(
             parsed.difficulty, len(self.difficulty_to_id)
@@ -470,6 +481,8 @@ class DINOv3ActionChunkPolicy:
             gripper_state=gripper_state,
             last_action=cached_action,
         )
+        if pending_text_cache is not None:
+            self._text_cache[text_key] = pending_text_cache
         return action.copy()
 
 
@@ -506,6 +519,31 @@ def _single_encoded_frame(
     return result
 
 
+def _single_text_feature(
+    value: Any,
+    expected_dim: int,
+    device: torch.device,
+    dtype: torch.dtype,
+) -> torch.Tensor:
+    if (
+        not isinstance(value, torch.Tensor)
+        or value.ndim != 2
+        or tuple(value.shape) != (1, expected_dim)
+    ):
+        shape = (
+            tuple(value.shape)
+            if isinstance(value, torch.Tensor)
+            else type(value).__name__
+        )
+        raise RuntimeError(
+            f"text encoder output must be [1,{expected_dim}], got {shape}"
+        )
+    result = value.detach().to(device=device, dtype=dtype)
+    if not bool(torch.isfinite(result.float()).all().item()):
+        raise RuntimeError("text encoder output contains NaN or Inf")
+    return result
+
+
 def _validate_id_map(value: Any, name: str) -> dict[str, int]:
     if not isinstance(value, Mapping):
         raise ValueError(f"vla_config {name} must be an object")
@@ -523,49 +561,124 @@ def _validate_id_map(value: Any, name: str) -> dict[str, int]:
     return result
 
 
-def load_policy(model_dir: str, device: str, dtype: str) -> DINOv3ActionChunkPolicy:
+def _build_local_clip_tokenizer(
+    tokenizer_cls: Any,
+    vocab_path: Path,
+    merges_path: Path,
+    clip_config: Mapping[str, Any],
+) -> Any:
+    """Build the bundled byte-BPE tokenizer across Transformers 4.x/5.x."""
+
+    text_config = clip_config.get("text_config")
+    if not isinstance(text_config, Mapping):
+        raise ValueError("clip_config.text_config must be an object")
+    try:
+        expected_vocab_size = int(text_config["vocab_size"])
+        expected_bos = int(text_config.get("bos_token_id", expected_vocab_size - 2))
+        expected_eos = int(text_config.get("eos_token_id", expected_vocab_size - 1))
+    except (KeyError, TypeError, ValueError) as exc:
+        raise ValueError(
+            "clip_config.text_config has an invalid tokenizer contract"
+        ) from exc
+    common = {"model_max_length": 77}
+    attempts = (
+        (
+            "transformers_v5",
+            {"vocab": str(vocab_path), "merges": str(merges_path)},
+        ),
+        (
+            "transformers_v4",
+            {
+                "vocab_file": str(vocab_path),
+                "merges_file": str(merges_path),
+            },
+        ),
+    )
+    failures: list[str] = []
+    for label, asset_kwargs in attempts:
+        try:
+            tokenizer = tokenizer_cls(**asset_kwargs, **common)
+            observed = {
+                "vocab_size": int(tokenizer.vocab_size),
+                "get_vocab_size": len(tokenizer.get_vocab()),
+                "bos_token_id": int(tokenizer.bos_token_id),
+                "eos_token_id": int(tokenizer.eos_token_id),
+                "pad_token_id": int(tokenizer.pad_token_id),
+                "unk_token_id": int(tokenizer.unk_token_id),
+            }
+        except (AttributeError, OSError, TypeError, ValueError) as exc:
+            failures.append(f"{label}: {type(exc).__name__}: {exc}")
+            continue
+        expected = {
+            "vocab_size": expected_vocab_size,
+            "get_vocab_size": expected_vocab_size,
+            "bos_token_id": expected_bos,
+            "eos_token_id": expected_eos,
+            "pad_token_id": expected_eos,
+            "unk_token_id": expected_eos,
+        }
+        mismatches = {
+            name: {"expected": expected[name], "actual": value}
+            for name, value in observed.items()
+            if value != expected[name]
+        }
+        if not mismatches:
+            return tokenizer
+        failures.append(f"{label}: {json.dumps(mismatches, sort_keys=True)}")
+    raise ValueError(
+        "Could not construct the bundled MetaCLIP tokenizer; " + " | ".join(failures)
+    )
+
+
+def load_policy(model_dir: str, device: str, dtype: str) -> MetaCLIPActionChunkPolicy:
     """Load a complete local checkpoint without any Hub/network fallback."""
     model_root = Path(model_dir).expanduser().resolve()
     config_path = model_root / "vla_config.json"
     weights_path = model_root / "model.safetensors"
-    runtime_path = model_root / "dinov3_action_chunk_model.py"
-    for path in (config_path, weights_path, runtime_path):
+    runtime_path = model_root / "metaclip_action_chunk_model.py"
+    vocab_path = model_root / "vocab.json"
+    merges_path = model_root / "merges.txt"
+    for path in (config_path, weights_path, runtime_path, vocab_path, merges_path):
         if not path.is_file():
             raise FileNotFoundError(
-                f"self-contained DINOv3 submission is missing {path.name}"
+                f"self-contained MetaCLIP submission is missing {path.name}"
             )
 
     config = json.loads(config_path.read_text(encoding="utf-8"))
-    if not isinstance(config, dict) or not isinstance(
-        config.get("vision_config"), dict
-    ):
-        raise ValueError("vla_config.json must contain an inline vision_config object")
+    if not isinstance(config, dict) or not isinstance(config.get("clip_config"), dict):
+        raise ValueError("vla_config.json must contain an inline clip_config object")
     if str(model_root) not in sys.path:
         sys.path.insert(0, str(model_root))
 
     # These imports are intentionally local. The adapter remains easy to test
     # with a fake model, and production has no model-Hub loading path.
     from safetensors.torch import load_file
-    from dinov3_action_chunk_model import (
-        DinoV3ActionChunkModel,
+    from transformers import CLIPTokenizer
+    from metaclip_action_chunk_model import (
+        MetaCLIPActionChunkModel,
         phase_vector,
-        text_vector,
     )
 
     torch_device = _resolve_device(device)
     torch_dtype = _resolve_dtype(dtype, torch_device)
-    model = DinoV3ActionChunkModel(config)
+    model = MetaCLIPActionChunkModel(config)
     state = load_file(str(weights_path), device="cpu")
     model.load_state_dict(state, strict=True)
     del state
-    return DINOv3ActionChunkPolicy(
+    tokenizer = _build_local_clip_tokenizer(
+        CLIPTokenizer,
+        vocab_path,
+        merges_path,
+        config["clip_config"],
+    )
+    return MetaCLIPActionChunkPolicy(
         model=model,
         config=config,
         device=torch_device,
         dtype=torch_dtype,
-        text_vector_fn=text_vector,
+        tokenizer=tokenizer,
         phase_vector_fn=phase_vector,
     )
 
 
-__all__ = ["DINOv3ActionChunkPolicy", "load_policy"]
+__all__ = ["MetaCLIPActionChunkPolicy", "load_policy"]

metaclip_action_chunk_model.py

@@ -0,0 +1,730 @@
+"""Self-contained MetaCLIP temporal action-chunk policy definition.
+
+This module deliberately contains no model-hub access.  A complete
+``clip_config`` dictionary is embedded in the policy configuration, so
+constructing :class:`MetaCLIPActionChunkModel` only creates modules.  Callers are
+responsible for loading a local state dict afterwards.
+
+The image, phase, and token helpers are shared by feature-cache creation,
+training, and the submission adapter.  Keeping those operations here prevents
+subtle train/deployment preprocessing drift.
+"""
+
+from __future__ import annotations
+
+import copy
+import math
+from collections.abc import Mapping
+from typing import Any
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import CLIPConfig, CLIPModel
+
+ACTION_DIM = 7
+DEFAULT_TEXT_DIM = 512
+DEFAULT_PHASE_DIM = 4
+DEFAULT_SPATIAL_HEADS = 8
+DEFAULT_DIFFICULTIES = ("low", "medium", "hard", "very_high")
+TEXT_FEATURE_VERSION = "metaclip_clip_bpe_projected_l2_text_v2"
+METACLIP_IMAGE_MEAN = (0.48145466, 0.4578275, 0.40821073)
+METACLIP_IMAGE_STD = (0.26862954, 0.26130258, 0.27577711)
+
+_REQUIRED_CONFIG_KEYS = (
+    "clip_config",
+    "image_size",
+    "spatial_grid",
+    "proprio_dim",
+    "text_dim",
+    "phase_dim",
+    "hidden_dim",
+    "history",
+    "action_chunk",
+    "ensemble_heads",
+    "dropout",
+    "task_to_id",
+    "difficulty_to_id",
+)
+
+
+def _require_plain_int(value: Any, name: str, *, minimum: int = 1) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
+        raise ValueError(f"{name} must be an integer >= {minimum}, got {value!r}")
+    return int(value)
+
+
+def _require_probability(value: Any, name: str) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise ValueError(f"{name} must be a number in [0, 1), got {value!r}")
+    value = float(value)
+    if not math.isfinite(value) or not 0.0 <= value < 1.0:
+        raise ValueError(f"{name} must be a finite number in [0, 1), got {value!r}")
+    return value
+
+
+def _validate_id_map(value: Any, name: str) -> dict[str, int]:
+    if not isinstance(value, Mapping) or not value:
+        raise ValueError(f"{name} must be a non-empty mapping of strings to IDs")
+    result: dict[str, int] = {}
+    for key, item in value.items():
+        if not isinstance(key, str) or not key.strip():
+            raise ValueError(f"{name} contains an invalid key: {key!r}")
+        if isinstance(item, bool) or not isinstance(item, int) or item < 0:
+            raise ValueError(f"{name}[{key!r}] must be a non-negative integer")
+        if key in result:
+            raise ValueError(f"{name} contains duplicate key {key!r}")
+        result[key] = int(item)
+    expected = list(range(len(result)))
+    actual = sorted(result.values())
+    if actual != expected:
+        raise ValueError(
+            f"{name} IDs must be unique and contiguous 0..{len(result) - 1}, got {actual}"
+        )
+    return result
+
+
+def validate_policy_config(config: Mapping[str, Any]) -> dict[str, Any]:
+    """Validate and return an isolated copy of a policy configuration.
+
+    Deployment/training metadata outside the architectural keys is retained,
+    but all fields that affect tensor shapes or preprocessing are checked.  The
+    returned object can therefore be safely stored on a model without later
+    mutations by the caller changing its behavior.
+    """
+
+    if not isinstance(config, Mapping):
+        raise TypeError(f"config must be a mapping, got {type(config).__name__}")
+    missing = [key for key in _REQUIRED_CONFIG_KEYS if key not in config]
+    if missing:
+        raise ValueError(f"policy config is missing required keys: {missing}")
+
+    validated = copy.deepcopy(dict(config))
+    clip_config = validated["clip_config"]
+    if not isinstance(clip_config, Mapping) or not clip_config:
+        raise ValueError("clip_config must be a non-empty mapping")
+    clip_config = copy.deepcopy(dict(clip_config))
+    if clip_config.get("model_type") != "clip":
+        raise ValueError(
+            "clip_config.model_type must be 'clip', got "
+            f"{clip_config.get('model_type')!r}"
+        )
+    if (
+        _require_plain_int(
+            clip_config.get("projection_dim"), "clip_config.projection_dim"
+        )
+        != DEFAULT_TEXT_DIM
+    ):
+        raise ValueError(
+            f"MetaCLIP projection_dim must be {DEFAULT_TEXT_DIM}, "
+            f"got {clip_config.get('projection_dim')!r}"
+        )
+
+    vision_config = clip_config.get("vision_config")
+    if not isinstance(vision_config, Mapping) or not vision_config:
+        raise ValueError("clip_config.vision_config must be a non-empty mapping")
+    vision_config = copy.deepcopy(dict(vision_config))
+    for key in (
+        "hidden_size",
+        "intermediate_size",
+        "image_size",
+        "patch_size",
+        "num_hidden_layers",
+        "num_attention_heads",
+    ):
+        if key not in vision_config:
+            raise ValueError(
+                f"clip_config.vision_config is missing required key {key!r}"
+            )
+        _require_plain_int(vision_config[key], f"clip_config.vision_config.{key}")
+    if vision_config.get("model_type") != "clip_vision_model":
+        raise ValueError(
+            "clip_config.vision_config.model_type must be 'clip_vision_model', got "
+            f"{vision_config.get('model_type')!r}"
+        )
+    if (
+        int(vision_config["image_size"]) != 224
+        or int(vision_config["patch_size"]) != 16
+    ):
+        raise ValueError(
+            "The audited MetaCLIP B/16 contract requires image_size=224 and "
+            f"patch_size=16, got {vision_config['image_size']!r} and "
+            f"{vision_config['patch_size']!r}"
+        )
+    if int(vision_config["hidden_size"]) % int(vision_config["num_attention_heads"]):
+        raise ValueError(
+            "vision_config.hidden_size must be divisible by "
+            "vision_config.num_attention_heads"
+        )
+    if "num_channels" in vision_config and int(vision_config["num_channels"]) != 3:
+        raise ValueError(
+            "only three-channel MetaCLIP vision configurations are supported"
+        )
+
+    text_config = clip_config.get("text_config")
+    if not isinstance(text_config, Mapping) or not text_config:
+        raise ValueError("clip_config.text_config must be a non-empty mapping")
+    text_config = copy.deepcopy(dict(text_config))
+    for key in (
+        "hidden_size",
+        "intermediate_size",
+        "max_position_embeddings",
+        "num_hidden_layers",
+        "num_attention_heads",
+        "vocab_size",
+    ):
+        if key not in text_config:
+            raise ValueError(f"clip_config.text_config is missing required key {key!r}")
+        _require_plain_int(text_config[key], f"clip_config.text_config.{key}")
+    if text_config.get("model_type") != "clip_text_model":
+        raise ValueError(
+            "clip_config.text_config.model_type must be 'clip_text_model', got "
+            f"{text_config.get('model_type')!r}"
+        )
+    if int(text_config["max_position_embeddings"]) != 77:
+        raise ValueError(
+            "MetaCLIP text max_position_embeddings must be 77, got "
+            f"{text_config['max_position_embeddings']!r}"
+        )
+    clip_config["vision_config"] = vision_config
+    clip_config["text_config"] = text_config
+    validated["clip_config"] = clip_config
+
+    image_size = _require_plain_int(validated["image_size"], "image_size")
+    if image_size != int(vision_config["image_size"]):
+        raise ValueError(
+            "policy image_size must match vision_config.image_size, got "
+            f"{image_size} and {vision_config['image_size']!r}"
+        )
+    patch_size = int(vision_config["patch_size"])
+    if image_size % patch_size:
+        raise ValueError(
+            f"image_size={image_size} must be divisible by MetaCLIP patch_size={patch_size}"
+        )
+    patch_side = image_size // patch_size
+    spatial_grid = _require_plain_int(validated["spatial_grid"], "spatial_grid")
+    if spatial_grid > patch_side:
+        raise ValueError(
+            f"spatial_grid={spatial_grid} cannot exceed the {patch_side}x{patch_side} "
+            "MetaCLIP input patch grid"
+        )
+
+    for key in (
+        "proprio_dim",
+        "text_dim",
+        "phase_dim",
+        "hidden_dim",
+        "history",
+        "action_chunk",
+        "ensemble_heads",
+    ):
+        validated[key] = _require_plain_int(validated[key], key)
+    if validated["text_dim"] != int(clip_config["projection_dim"]):
+        raise ValueError(
+            "text_dim must equal clip_config.projection_dim, got "
+            f"{validated['text_dim']} and {clip_config['projection_dim']!r}"
+        )
+    if validated["phase_dim"] != DEFAULT_PHASE_DIM:
+        raise ValueError(
+            f"phase_dim must be {DEFAULT_PHASE_DIM} for phase_vector(), "
+            f"got {validated['phase_dim']}"
+        )
+    if validated["hidden_dim"] % DEFAULT_SPATIAL_HEADS:
+        raise ValueError(
+            f"hidden_dim must be divisible by {DEFAULT_SPATIAL_HEADS} spatial heads"
+        )
+    validated["dropout"] = _require_probability(validated["dropout"], "dropout")
+    validated["task_to_id"] = _validate_id_map(validated["task_to_id"], "task_to_id")
+    validated["difficulty_to_id"] = _validate_id_map(
+        validated["difficulty_to_id"], "difficulty_to_id"
+    )
+
+    if "action_dim" in validated and validated["action_dim"] != ACTION_DIM:
+        raise ValueError(
+            f"action_dim must be {ACTION_DIM}, got {validated['action_dim']!r}"
+        )
+    if "rgb_dim" in validated and validated["rgb_dim"] != 3:
+        raise ValueError(f"rgb_dim must be 3, got {validated['rgb_dim']!r}")
+    if (
+        "text_feature_version" in validated
+        and validated["text_feature_version"] != TEXT_FEATURE_VERSION
+    ):
+        raise ValueError(
+            f"text_feature_version must be {TEXT_FEATURE_VERSION!r}, got "
+            f"{validated['text_feature_version']!r}"
+        )
+    return validated
+
+
+def phase_vector(step: int, horizon: int) -> "np.ndarray":
+    """Encode episode progress using the exact train/runtime four-vector."""
+
+    if isinstance(step, bool) or not isinstance(step, (int, np.integer)):
+        raise ValueError(f"step must be an integer, got {step!r}")
+    if isinstance(horizon, bool) or not isinstance(horizon, (int, np.integer)):
+        raise ValueError(f"horizon must be an integer, got {horizon!r}")
+    horizon_f = max(float(horizon), 1.0)
+    progress = float(np.clip(float(step) / horizon_f, 0.0, 1.0))
+    return np.asarray(
+        [
+            progress,
+            1.0 - progress,
+            math.sin(math.pi * progress),
+            math.cos(math.pi * progress),
+        ],
+        dtype=np.float32,
+    )
+
+
+def _images_to_nchw_rgb(images: torch.Tensor) -> torch.Tensor:
+    if not isinstance(images, torch.Tensor):
+        raise TypeError(f"images must be a torch.Tensor, got {type(images).__name__}")
+    if images.ndim != 4:
+        raise ValueError(
+            f"expected a four-dimensional image tensor, got {tuple(images.shape)}"
+        )
+
+    # Prefer an unambiguous channel-first interpretation, then NHWC.  Normal
+    # robotics images are 224x224, so both layouts are unambiguous in practice.
+    if images.shape[1] in (1, 3, 4) and images.shape[-1] not in (1, 3, 4):
+        nchw = images
+    elif images.shape[-1] in (1, 3, 4):
+        nchw = images.permute(0, 3, 1, 2)
+    elif images.shape[1] in (1, 3, 4):
+        nchw = images
+    else:
+        raise ValueError(
+            f"cannot determine image channels for shape {tuple(images.shape)}"
+        )
+
+    if nchw.shape[1] == 1:
+        nchw = nchw.repeat(1, 3, 1, 1)
+    elif nchw.shape[1] == 4:
+        nchw = nchw[:, :3]
+    if nchw.shape[1] != 3:
+        raise ValueError(
+            f"expected one, three, or four image channels, got {nchw.shape[1]}"
+        )
+    return nchw
+
+
+def images_to_unit_rgb(images: torch.Tensor, image_size: int = 224) -> torch.Tensor:
+    """Convert NHWC/NCHW uint8-like images to resized NCHW RGB in ``[0, 1]``."""
+
+    image_size = _require_plain_int(image_size, "image_size")
+    rgb = _images_to_nchw_rgb(images).float()
+    if rgb.numel() and float(rgb.detach().amax().cpu()) > 2.0:
+        rgb = rgb / 255.0
+    if not bool(torch.isfinite(rgb).all().detach().cpu()):
+        raise ValueError("images contain NaN or Inf")
+    if tuple(rgb.shape[-2:]) != (image_size, image_size):
+        rgb = F.interpolate(
+            rgb,
+            size=(image_size, image_size),
+            mode="bicubic",
+            align_corners=False,
+            antialias=True,
+        )
+    return rgb
+
+
+def normalize_images(images: torch.Tensor, image_size: int = 224) -> torch.Tensor:
+    """Prepare image pixels for the MetaCLIP vision encoder."""
+
+    rgb = images_to_unit_rgb(images, image_size=image_size)
+    mean = rgb.new_tensor(METACLIP_IMAGE_MEAN).view(1, 3, 1, 1)
+    std = rgb.new_tensor(METACLIP_IMAGE_STD).view(1, 3, 1, 1)
+    return (rgb - mean) / std
+
+
+def rgb_grid_tokens(
+    images: torch.Tensor, spatial_grid: int, image_size: int = 224
+) -> torch.Tensor:
+    """Return global RGB plus a row-major spatial grid, shaped ``[B,1+G²,3]``."""
+
+    spatial_grid = _require_plain_int(spatial_grid, "spatial_grid")
+    rgb = images_to_unit_rgb(images, image_size=image_size)
+    global_rgb = rgb.mean(dim=(-2, -1)).unsqueeze(1)
+    grid_rgb = F.adaptive_avg_pool2d(rgb, (spatial_grid, spatial_grid))
+    grid_rgb = grid_rgb.flatten(2).transpose(1, 2)
+    return torch.cat([global_rgb, grid_rgb], dim=1)
+
+
+def pool_metaclip_tokens(
+    hidden_states: torch.Tensor,
+    spatial_grid: int,
+) -> torch.Tensor:
+    """Pool MetaCLIP patch tokens to ``G x G`` and retain the CLS token."""
+
+    spatial_grid = _require_plain_int(spatial_grid, "spatial_grid")
+    if not isinstance(hidden_states, torch.Tensor):
+        raise TypeError("hidden_states must be a torch.Tensor")
+    if hidden_states.ndim != 3 or hidden_states.shape[1] <= 1:
+        raise ValueError(
+            f"unexpected MetaCLIP output shape: {tuple(hidden_states.shape)}"
+        )
+    cls_token = hidden_states[:, :1]
+    patches = hidden_states[:, 1:]
+    side = math.isqrt(int(patches.shape[1]))
+    if side * side != int(patches.shape[1]):
+        raise ValueError(f"MetaCLIP patch count {patches.shape[1]} is not a square")
+    patches = patches.transpose(1, 2).reshape(
+        patches.shape[0], patches.shape[2], side, side
+    )
+    patches = F.adaptive_avg_pool2d(patches, (spatial_grid, spatial_grid))
+    patches = patches.flatten(2).transpose(1, 2)
+    return torch.cat([cls_token, patches], dim=1)
+
+
+class MetaCLIPActionChunkHead(nn.Module):
+    """Task-conditioned spatial pooling followed by a short temporal policy."""
+
+    def __init__(
+        self,
+        *,
+        vision_dim: int,
+        proprio_dim: int,
+        text_dim: int,
+        phase_dim: int,
+        num_tasks: int,
+        num_difficulties: int,
+        hidden_dim: int = 256,
+        history: int = 4,
+        action_chunk: int = 8,
+        ensemble_heads: int = 3,
+        dropout: float = 0.10,
+    ) -> None:
+        super().__init__()
+        self.vision_dim = _require_plain_int(vision_dim, "vision_dim")
+        self.proprio_dim = _require_plain_int(proprio_dim, "proprio_dim")
+        self.text_dim = _require_plain_int(text_dim, "text_dim")
+        self.phase_dim = _require_plain_int(phase_dim, "phase_dim")
+        self.hidden_dim = _require_plain_int(hidden_dim, "hidden_dim")
+        self.history = _require_plain_int(history, "history")
+        self.action_chunk = _require_plain_int(action_chunk, "action_chunk")
+        self.ensemble_heads = _require_plain_int(ensemble_heads, "ensemble_heads")
+        num_tasks = _require_plain_int(num_tasks, "num_tasks")
+        num_difficulties = _require_plain_int(num_difficulties, "num_difficulties")
+        dropout = _require_probability(dropout, "dropout")
+        if self.hidden_dim % DEFAULT_SPATIAL_HEADS:
+            raise ValueError(
+                f"hidden_dim must be divisible by {DEFAULT_SPATIAL_HEADS} spatial heads"
+            )
+
+        self.vision_proj = nn.Sequential(
+            nn.LayerNorm(self.vision_dim), nn.Linear(self.vision_dim, self.hidden_dim)
+        )
+        self.rgb_proj = nn.Sequential(nn.Linear(3, self.hidden_dim), nn.SiLU())
+        self.proprio_proj = nn.Sequential(
+            nn.LayerNorm(self.proprio_dim + self.phase_dim),
+            nn.Linear(self.proprio_dim + self.phase_dim, self.hidden_dim),
+            nn.SiLU(),
+            nn.Dropout(dropout),
+        )
+        self.text_proj = nn.Sequential(
+            nn.LayerNorm(self.text_dim),
+            nn.Linear(self.text_dim, self.hidden_dim),
+            nn.SiLU(),
+        )
+        # The last row of each embedding is the trained unknown/fallback ID.
+        self.task_embedding = nn.Embedding(num_tasks + 1, self.hidden_dim)
+        self.difficulty_embedding = nn.Embedding(num_difficulties + 1, self.hidden_dim)
+        self.task_scale = nn.Parameter(torch.tensor(0.5))
+        self.difficulty_scale = nn.Parameter(torch.tensor(0.25))
+        self.condition_norm = nn.LayerNorm(self.hidden_dim)
+        self.spatial_attention = nn.MultiheadAttention(
+            embed_dim=self.hidden_dim,
+            num_heads=DEFAULT_SPATIAL_HEADS,
+            dropout=dropout,
+            batch_first=True,
+        )
+        self.frame_fusion = nn.Sequential(
+            nn.Linear(self.hidden_dim * 2, self.hidden_dim),
+            nn.SiLU(),
+            nn.LayerNorm(self.hidden_dim),
+            nn.Dropout(dropout),
+        )
+        self.temporal_gru = nn.GRU(
+            input_size=self.hidden_dim,
+            hidden_size=self.hidden_dim,
+            num_layers=2,
+            dropout=dropout,
+            batch_first=True,
+        )
+        self.output_heads = nn.ModuleList(
+            [
+                nn.Sequential(
+                    nn.LayerNorm(self.hidden_dim),
+                    nn.Linear(self.hidden_dim, self.hidden_dim),
+                    nn.SiLU(),
+                    nn.Dropout(dropout),
+                    nn.Linear(self.hidden_dim, self.action_chunk * ACTION_DIM),
+                )
+                for _ in range(self.ensemble_heads)
+            ]
+        )
+
+    def _validate_inputs(
+        self,
+        visual_tokens: torch.Tensor,
+        rgb_tokens: torch.Tensor,
+        proprio: torch.Tensor,
+        phase: torch.Tensor,
+        text_features: torch.Tensor,
+        task_ids: torch.Tensor,
+        difficulty_ids: torch.Tensor,
+    ) -> tuple[int, int, int]:
+        if visual_tokens.ndim != 4:
+            raise ValueError(
+                f"visual_tokens must have shape [B,T,V,D], got {tuple(visual_tokens.shape)}"
+            )
+        batch, timesteps, token_count, vision_dim = visual_tokens.shape
+        if timesteps != self.history:
+            raise ValueError(f"expected history={self.history}, got {timesteps}")
+        if vision_dim != self.vision_dim:
+            raise ValueError(f"expected vision_dim={self.vision_dim}, got {vision_dim}")
+        if rgb_tokens.shape != (batch, timesteps, token_count, 3):
+            raise ValueError(
+                "rgb_tokens must align with visual_tokens and end in RGB, got "
+                f"{tuple(rgb_tokens.shape)}"
+            )
+        if proprio.shape != (batch, timesteps, self.proprio_dim):
+            raise ValueError(
+                f"proprio must have shape {(batch, timesteps, self.proprio_dim)}, "
+                f"got {tuple(proprio.shape)}"
+            )
+        if phase.shape != (batch, timesteps, self.phase_dim):
+            raise ValueError(
+                f"phase must have shape {(batch, timesteps, self.phase_dim)}, "
+                f"got {tuple(phase.shape)}"
+            )
+        if text_features.shape != (batch, self.text_dim):
+            raise ValueError(
+                f"text_features must have shape {(batch, self.text_dim)}, "
+                f"got {tuple(text_features.shape)}"
+            )
+        for name, ids in (("task_ids", task_ids), ("difficulty_ids", difficulty_ids)):
+            if ids.shape != (batch,):
+                raise ValueError(
+                    f"{name} must have shape {(batch,)}, got {tuple(ids.shape)}"
+                )
+            if ids.dtype not in (torch.int32, torch.int64):
+                raise ValueError(
+                    f"{name} must contain integer IDs, got dtype={ids.dtype}"
+                )
+        return batch, timesteps, token_count
+
+    def forward_cached(
+        self,
+        visual_tokens: torch.Tensor,
+        rgb_tokens: torch.Tensor,
+        proprio: torch.Tensor,
+        phase: torch.Tensor,
+        text_features: torch.Tensor,
+        task_ids: torch.Tensor,
+        difficulty_ids: torch.Tensor,
+    ) -> torch.Tensor:
+        """Return raw action logits shaped ``[ensemble, batch, chunk, 7]``."""
+
+        batch, timesteps, token_count = self._validate_inputs(
+            visual_tokens,
+            rgb_tokens,
+            proprio,
+            phase,
+            text_features,
+            task_ids,
+            difficulty_ids,
+        )
+        visual = self.vision_proj(visual_tokens) + self.rgb_proj(rgb_tokens)
+        state = self.proprio_proj(torch.cat([proprio, phase], dim=-1))
+        text = self.text_proj(text_features)
+        task = self.task_embedding(task_ids)
+        difficulty = self.difficulty_embedding(difficulty_ids)
+        condition = self.condition_norm(
+            state
+            + text[:, None]
+            + torch.tanh(self.task_scale) * task[:, None]
+            + torch.tanh(self.difficulty_scale) * difficulty[:, None]
+        )
+
+        flat_visual = visual.reshape(batch * timesteps, token_count, self.hidden_dim)
+        flat_query = condition.reshape(batch * timesteps, 1, self.hidden_dim)
+        attended, _ = self.spatial_attention(
+            flat_query, flat_visual, flat_visual, need_weights=False
+        )
+        attended = attended.reshape(batch, timesteps, self.hidden_dim)
+        frames = self.frame_fusion(torch.cat([attended, condition], dim=-1))
+        temporal, _ = self.temporal_gru(frames)
+        final = temporal[:, -1]
+        outputs = [
+            head(final).reshape(batch, self.action_chunk, ACTION_DIM)
+            for head in self.output_heads
+        ]
+        return torch.stack(outputs, dim=0)
+
+    def forward(
+        self,
+        visual_tokens: torch.Tensor,
+        rgb_tokens: torch.Tensor,
+        proprio: torch.Tensor,
+        phase: torch.Tensor,
+        text_features: torch.Tensor,
+        task_ids: torch.Tensor,
+        difficulty_ids: torch.Tensor,
+    ) -> torch.Tensor:
+        return self.forward_cached(
+            visual_tokens,
+            rgb_tokens,
+            proprio,
+            phase,
+            text_features,
+            task_ids,
+            difficulty_ids,
+        )
+
+
+class MetaCLIPActionChunkModel(nn.Module):
+    """Complete submission model containing frozen MetaCLIP and the policy head."""
+
+    def __init__(self, config: Mapping[str, Any]) -> None:
+        super().__init__()
+        self.policy_config = validate_policy_config(config)
+        self.spatial_grid = int(self.policy_config["spatial_grid"])
+
+        # Offline construction only: this creates a model from the embedded
+        # architecture.  It never resolves a repository or downloads weights.
+        clip_config = CLIPConfig.from_dict(self.policy_config["clip_config"])
+        self.clip = CLIPModel(clip_config)
+        self.head = MetaCLIPActionChunkHead(
+            vision_dim=int(clip_config.vision_config.hidden_size),
+            proprio_dim=int(self.policy_config["proprio_dim"]),
+            text_dim=int(self.policy_config["text_dim"]),
+            phase_dim=int(self.policy_config["phase_dim"]),
+            num_tasks=len(self.policy_config["task_to_id"]),
+            num_difficulties=len(self.policy_config["difficulty_to_id"]),
+            hidden_dim=int(self.policy_config["hidden_dim"]),
+            history=int(self.policy_config["history"]),
+            action_chunk=int(self.policy_config["action_chunk"]),
+            ensemble_heads=int(self.policy_config["ensemble_heads"]),
+            dropout=float(self.policy_config["dropout"]),
+        )
+
+    def freeze_backbone(self) -> None:
+        """Freeze both MetaCLIP towers and keep them in inference mode."""
+
+        self.clip.requires_grad_(False)
+        self.clip.eval()
+
+    def train(self, mode: bool = True) -> "MetaCLIPActionChunkModel":
+        # A caller may train the complete wrapper for convenience.  If the
+        # backbone has been frozen, do not accidentally switch it back to train
+        # mode through nn.Module.train() recursion.
+        super().train(mode)
+        if not any(parameter.requires_grad for parameter in self.clip.parameters()):
+            self.clip.eval()
+        return self
+
+    def encode_images(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+        """Encode an image batch into aligned MetaCLIP and raw-RGB tokens."""
+
+        image_size = int(self.policy_config["image_size"])
+        rgb_tokens = rgb_grid_tokens(
+            images, spatial_grid=self.spatial_grid, image_size=image_size
+        )
+        pixels = normalize_images(images, image_size=image_size)
+        vision_parameter = next(self.clip.vision_model.parameters())
+        pixels = pixels.to(device=vision_parameter.device, dtype=vision_parameter.dtype)
+        rgb_tokens = rgb_tokens.to(
+            device=vision_parameter.device, dtype=vision_parameter.dtype
+        )
+        hidden = self.clip.vision_model(pixel_values=pixels).last_hidden_state
+        hidden = self.clip.vision_model.post_layernorm(hidden)
+        return (
+            pool_metaclip_tokens(hidden, self.spatial_grid),
+            rgb_tokens,
+        )
+
+    def encode_text(
+        self, input_ids: torch.Tensor, attention_mask: torch.Tensor
+    ) -> torch.Tensor:
+        """Return normalized projected MetaCLIP instruction embeddings."""
+
+        parameter = next(self.clip.text_model.parameters())
+        input_ids = input_ids.to(device=parameter.device)
+        attention_mask = attention_mask.to(device=parameter.device)
+        outputs = self.clip.text_model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            return_dict=True,
+        )
+        features = self.clip.text_projection(outputs.pooler_output)
+        return F.normalize(features.float(), dim=-1).to(dtype=parameter.dtype)
+
+    def forward_cached(
+        self,
+        visual_tokens: torch.Tensor,
+        rgb_tokens: torch.Tensor,
+        proprio: torch.Tensor,
+        phase: torch.Tensor,
+        text_features: torch.Tensor,
+        task_ids: torch.Tensor,
+        difficulty_ids: torch.Tensor,
+    ) -> torch.Tensor:
+        return self.head.forward_cached(
+            visual_tokens,
+            rgb_tokens,
+            proprio,
+            phase,
+            text_features,
+            task_ids,
+            difficulty_ids,
+        )
+
+    def forward(
+        self,
+        visual_tokens: torch.Tensor,
+        rgb_tokens: torch.Tensor,
+        proprio: torch.Tensor,
+        phase: torch.Tensor,
+        text_features: torch.Tensor,
+        task_ids: torch.Tensor,
+        difficulty_ids: torch.Tensor,
+    ) -> torch.Tensor:
+        return self.forward_cached(
+            visual_tokens,
+            rgb_tokens,
+            proprio,
+            phase,
+            text_features,
+            task_ids,
+            difficulty_ids,
+        )
+
+
+# Compatibility aliases make reference checkpoints/code easy to compare while
+# retaining descriptive names in the new trainer.
+CompetitivePolicyHead = MetaCLIPActionChunkHead
+CompetitiveVLAModel = MetaCLIPActionChunkModel
+
+
+__all__ = [
+    "ACTION_DIM",
+    "DEFAULT_DIFFICULTIES",
+    "DEFAULT_PHASE_DIM",
+    "DEFAULT_SPATIAL_HEADS",
+    "DEFAULT_TEXT_DIM",
+    "METACLIP_IMAGE_MEAN",
+    "METACLIP_IMAGE_STD",
+    "TEXT_FEATURE_VERSION",
+    "CompetitivePolicyHead",
+    "CompetitiveVLAModel",
+    "MetaCLIPActionChunkHead",
+    "MetaCLIPActionChunkModel",
+    "images_to_unit_rgb",
+    "normalize_images",
+    "phase_vector",
+    "pool_metaclip_tokens",
+    "rgb_grid_tokens",
+    "validate_policy_config",
+]

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b833cd1cee8d6070d162883e211cb13cdb455ed74633b5b06de199b2035be5ef
-size 1219514848
+oid sha256:c8cda3c02b88b7b1a1e164e66bf4168a25a5b022bf3466f791108c1185974217
+size 605614380

vla_config.json

@@ -10,8 +10,104 @@
     "difficulty",
     "horizon"
   ],
-  "backbone_id_for_provenance": "facebook/dinov3-vitl16-pretrain-lvd1689m",
-  "difficulty_condition_dropout": 0.45,
+  "backbone_id_for_provenance": "facebook/metaclip-b16-fullcc2.5b",
+  "clip_config": {
+    "_name_or_path": "facebook/metaclip-b16-fullcc2.5b",
+    "architectures": [
+      "CLIPModel"
+    ],
+    "chunk_size_feed_forward": 0,
+    "dtype": "float32",
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_factor": 1.0,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "logit_scale_init_value": 2.6592,
+    "model_type": "clip",
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "problem_type": null,
+    "projection_dim": 512,
+    "return_dict": true,
+    "text_config": {
+      "_name_or_path": "",
+      "architectures": null,
+      "attention_dropout": 0.0,
+      "bos_token_id": 49406,
+      "chunk_size_feed_forward": 0,
+      "dtype": "float32",
+      "eos_token_id": 49407,
+      "heads": 8,
+      "hidden_act": "quick_gelu",
+      "hidden_size": 512,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_factor": 1.0,
+      "initializer_range": 0.02,
+      "intermediate_size": 2048,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-05,
+      "layers": 12,
+      "max_position_embeddings": 77,
+      "model_type": "clip_text_model",
+      "num_attention_heads": 8,
+      "num_hidden_layers": 12,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "pad_token_id": 1,
+      "problem_type": null,
+      "projection_dim": 512,
+      "return_dict": true,
+      "vocab_size": 49408
+    },
+    "transformers_version": "5.4.0",
+    "vision_config": {
+      "_name_or_path": "",
+      "architectures": null,
+      "attention_dropout": 0.0,
+      "chunk_size_feed_forward": 0,
+      "dtype": "float32",
+      "hidden_act": "quick_gelu",
+      "hidden_size": 768,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "image_size": 224,
+      "initializer_factor": 1.0,
+      "initializer_range": 0.02,
+      "intermediate_size": 3072,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-05,
+      "model_type": "clip_vision_model",
+      "num_attention_heads": 12,
+      "num_channels": 3,
+      "num_hidden_layers": 12,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "patch_size": 16,
+      "problem_type": null,
+      "projection_dim": 512,
+      "return_dict": true
+    }
+  },
+  "difficulty_condition_dropout": 0.5,
   "difficulty_to_id": {
     "low": 0,
     "medium": 1
@@ -29,17 +125,17 @@
   "history": 4,
   "image_size": 224,
   "license": {
-    "file": "LICENSE.md",
-    "id": "dinov3-license",
-    "source_sha256": "25d122eb8f5b880fd23c736fb6ea8018ee45c12237e00b8a86d14c653904999e"
+    "file": "CC-BY-NC-4.0.txt",
+    "id": "cc-by-nc-4.0",
+    "source_sha256": "41003d4a74749c0220e33dd415042164b5a1093ed401f36277234f772d22d3d0"
   },
-  "model_type": "frozen_dinov3_rgb_temporal_action_chunk_bc",
+  "model_type": "frozen_metaclip_vision_text_temporal_action_chunk_bc",
   "phase_dim": 4,
   "proprio_dim": 25,
   "rgb_dim": 3,
-  "schema_version": "flock_robotics_dinov3_action_chunk_v1",
+  "schema_version": "flock_robotics_metaclip_action_chunk_v2",
   "spatial_grid": 8,
-  "task_condition_dropout": 0.3,
+  "task_condition_dropout": 0.5,
   "task_to_id": {
     "lift_cube": 0,
     "pick_place_bread": 1,
@@ -49,89 +145,13 @@
     "stack_blocks": 5
   },
   "temporal_ensemble_decay": 0.55,
-  "text_dim": 128,
-  "text_feature_version": "signed_hash_subtokens_v2",
-  "unknown_condition_strategy": "trained_fallback_embedding",
-  "vision_config": {
-    "_name_or_path": "facebook/dinov3-vitl16-pretrain-lvd1689m",
-    "apply_layernorm": true,
-    "architectures": [
-      "DINOv3ViTModel"
-    ],
-    "attention_dropout": 0.0,
-    "chunk_size_feed_forward": 0,
-    "drop_path_rate": 0.0,
-    "dtype": "float32",
-    "hidden_act": "gelu",
-    "hidden_size": 1024,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "image_size": 224,
-    "initializer_range": 0.02,
-    "intermediate_size": 4096,
-    "is_encoder_decoder": false,
-    "key_bias": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-05,
-    "layerscale_value": 1.0,
-    "mlp_bias": true,
-    "model_type": "dinov3_vit",
-    "num_attention_heads": 16,
-    "num_channels": 3,
-    "num_hidden_layers": 24,
-    "num_register_tokens": 4,
-    "out_features": [
-      "stage24"
-    ],
-    "out_indices": [
-      24
-    ],
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "patch_size": 16,
-    "pos_embed_jitter": null,
-    "pos_embed_rescale": 2.0,
-    "pos_embed_shift": null,
-    "problem_type": null,
-    "proj_bias": true,
-    "query_bias": true,
-    "reshape_hidden_states": true,
-    "return_dict": true,
-    "rope_theta": 100.0,
-    "stage_names": [
-      "stem",
-      "stage1",
-      "stage2",
-      "stage3",
-      "stage4",
-      "stage5",
-      "stage6",
-      "stage7",
-      "stage8",
-      "stage9",
-      "stage10",
-      "stage11",
-      "stage12",
-      "stage13",
-      "stage14",
-      "stage15",
-      "stage16",
-      "stage17",
-      "stage18",
-      "stage19",
-      "stage20",
-      "stage21",
-      "stage22",
-      "stage23",
-      "stage24"
-    ],
-    "transformers_version": "5.4.0",
-    "use_gated_mlp": false,
-    "value_bias": true
-  }
+  "text_dim": 512,
+  "text_feature_version": "metaclip_clip_bpe_projected_l2_text_v2",
+  "tokenizer": {
+    "merges_file": "merges.txt",
+    "model_max_length": 77,
+    "type": "CLIPTokenizer",
+    "vocab_file": "vocab.json"
+  },
+  "unknown_condition_strategy": "trained_fallback_embedding"
 }