Publish validated multimodal EmbeddingGemma 2 Apple Core AI conversion

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+audio.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+embedding.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+vision.aimodel/main.mlirb filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -0,0 +1,5 @@
+{
+    "embedding_dimension": 768,
+    "pooling_mode": "mean",
+    "include_prompt": true
+}

2_Normalize/config.json

@@ -0,0 +1,4 @@
+{
+    "module_input_name": "sentence_embedding",
+    "module_output_name": "sentence_embedding"
+}

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Alibaba Cloud
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,4 @@
+EmbeddingGemma 2 source weights and original model documentation by Google DeepMind, distributed under Apache-2.0.
+Host tokenization and preprocessing use Hugging Face Transformers under Apache-2.0.
+Core AI conversion and adapter by maxffarrell, 2026, Apache-2.0.
+See UPSTREAM_README.md and SOURCE.json for attribution and pinned provenance.

README.md

@@ -0,0 +1,74 @@
+---
+license: apache-2.0
+base_model: google/embeddinggemma-2
+pipeline_tag: feature-extraction
+library_name: coreai
+tags:
+- coreai
+- apple
+- embeddings
+- multimodal
+- image-feature-extraction
+- audio-feature-extraction
+- video-feature-extraction
+- experimental
+---
+
+# EmbeddingGemma 2 — Apple Core AI
+
+Apple Core AI `.aimodel` conversion of [google/embeddinggemma-2](https://huggingface.co/google/embeddinggemma-2/tree/914f7f89142e33e77833254d9c9b90c3cef7303b), revision `914f7f89142e33e77833254d9c9b90c3cef7303b`, Apache-2.0. This is the new 740M multimodal model, distinct from EmbeddingGemma 300M.
+
+Supports text/code, images, video frames, audio, and interleaved combinations within the limits below. Returns normalized 768-dimensional embeddings. All neural inference, projection, masked mean pooling and initial L2 normalization run in Core AI. The host uses Transformers for tokenization/media preprocessing and NumPy for feature placement and optional truncated-vector normalization. No source PyTorch checkpoint is needed for inference.
+
+## Limits and platform
+
+- Batch size one; **1,024 combined text and media tokens**, including special tokens. The source supports 8,192; this conversion is a fixed smaller export. Oversized inputs raise an error instead of being truncated.
+- Vision: 2,520 patches per image/frame, at most 280 soft tokens. Preserves the upstream aspect-ratio-aware image processor and padding. Each image/frame is processed sequentially; total context capacity still applies.
+- Audio: one mono 16 kHz clip per `encode` call, maximum **11.2 seconds** (1,120 mel frames / 280 soft tokens). No resampling in the adapter. Longer clips must be handled separately by the caller; this does not implement the source's minute-scale audio context.
+- Video accepts an already sampled NumPy `[frames,height,width,channels]` array. Validation used two frames. Decode and sample video files before calling the adapter; no file decoding, temporal sampling or audio-track extraction is implemented by this adapter. The upstream per-frame image processing is preserved.
+- Text task prefixes are preserved via `prompt_name`, e.g. `SearchQuery`, `Document`, `CodeRetrieval`. For mixed content, manually supply placeholder text (`<|image|>`, `<|audio|>`, `<|video|>`) in the desired order and omit `prompt_name`.
+- Optional Matryoshka truncation re-normalizes the vector; dimensions 128, 256 and 512 were exercised.
+- FP32 inference with the source weights promoted from BF16; assets total approximately 3 GB. Tested on macOS 27.0.1 / Apple Silicon with default Core AI specialization. No independent iOS, ANE or GPU placement claim.
+- Experimental community conversion. Numerical fixtures are conversion tests, not a retrieval-quality benchmark.
+
+## Download and run
+
+```bash
+hf download maxffarrell/embeddinggemma-2-CoreAI --local-dir embeddinggemma-2-CoreAI
+cd embeddinggemma-2-CoreAI
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+```python
+import asyncio
+import numpy as np
+from PIL import Image
+from coreai_embeddinggemma2 import EmbeddingGemma2CoreAI
+
+async def main():
+    model = await EmbeddingGemma2CoreAI.load(".")
+    query = await model.encode("What causes the northern lights?", prompt_name="SearchQuery")
+    document = await model.encode("Charged particles from the sun cause auroras.", prompt_name="Document")
+    print(float(query @ document))
+    image = await model.encode(images=Image.open("photo.jpg"))
+    audio = await model.encode(audio=np.zeros(16000, dtype=np.float32), sampling_rate=16000)
+    small = await model.encode("An example sentence", truncate_dim=256)
+
+asyncio.run(main())
+```
+
+## Validation
+
+`validation.json`: **21 cases** against the full pinned PyTorch FP32 model with native-length source inputs, testing task prefixes, code, Spanish, Japanese, short/long/near-capacity text, images, mixed text/images, two images, short/long/limit audio, text/audio, video, text/video and all four modalities together. Worst elementwise error: **9.76026058e-07**; minimum cosine similarity: **0.999999881**. Predeclared final limits: cosine > 0.99999 and maximum elementwise error < 0.00005, plus finite unit vectors.
+
+`smoke.json`: separate source-checkpoint-free native runtime smoke in five text/media cases with PyTorch neural model loading explicitly disabled, plus invalid-input rejection. Task-independent retrieval accuracy has not been benchmarked.
+
+## Reproduce
+
+From a parent directory containing this bundle under `bundles/embeddinggemma-2-CoreAI`, download the pinned source into `upstream/embeddinggemma-2`, install `requirements.txt`, and run `convert_gemma2.py`, `validate_gemma2.py`, then `smoke_gemma2.py`. The validation/smoke scripts default to that directory layout. The converter uses the exact official Transformers commit in requirements because stable 5.17 predates support. `coreai-torch` 0.4.3 warns it was validated through PyTorch 2.13; this export was tested on 2.14.
+
+Graphs: `embedding.aimodel` (text backbone + projection + pooling + normalization), `vision.aimodel` (vision encoder + projection + valid-token mask), `audio.aimodel` (audio encoder + projection + valid-token mask). Integer inputs are int32. `manifest.json` declares signatures, static shapes and output names. An equivalent slice/stack decomposition replaces unsupported `aten.unfold` in audio; the source weights are unchanged. The text mask preserves the source's inclusive bidirectional sliding window.
+
+`SOURCE.json` records the verified upstream weight hash; `SHA256SUMS` covers every release file except itself. Original model documentation is in `UPSTREAM_README.md`. For Clef Flash, an existing Core AI conversion was found and verified to use current source weight hashes; see `RELATED_MODEL_RESEARCH.md`.

RELATED_MODEL_RESEARCH.md

@@ -0,0 +1,9 @@
+# Clef / EmbeddingGemma follow-up, 2026-10-11
+
+Hub author search found Cloudflare/clef, Cloudflare/clef-flash, Cloudflare/clef-omni. Hub searches for clef-CoreAI and embeddinggemma-2-CoreAI, plus internet searches, found one Clef Flash Core AI repository and no EmbeddingGemma 2 Core AI repository. Search is evidence of discoverability, not proof none exists anywhere.
+
+Existing public Clef Flash conversion: https://huggingface.co/mlboydaisuke/clef-flash-CoreAI at 220ac149ed1136465f7d9dac93aaf0dfc55d6b73. Source revision 17f0b0ad64efb65d273590632833508766b2aae6. Compared every source safetensors file's LFS SHA256 with Cloudflare/clef-flash current revision 8b2e5fd17c09fd49bc2880b805d5323ec7ff4ff3: all five match, including joint_head. Assets are .aimodel directories. This check verifies published provenance, not an independent runtime audit. Runtime: https://github.com/john-rocky/coreai-model-zoo/tree/main/apps/ClefFlash
+
+Clef 27B has 12 source shards and exceeds a safe conversion budget on this 64 GiB Mac once compiler copies are included. Clef Omni at 0db1cd2607d76a7bdb2a382f659e7b313079f84b has ~70.5 GB source weights, more than installed RAM. Neither was downloaded or attempted. These are capacity preflight exclusions, not observed converter failures.
+
+The old embeddinggemma-300m-CoreAI repositories are not EmbeddingGemma 2. New source google/embeddinggemma-2 at 914f7f89142e33e77833254d9c9b90c3cef7303b is Apache-2.0, 740M multimodal. Transformers 5.17 lacked support. Installed official upstream commit 536ecc007387a50e77603bb5d92100e9b07514cc (5.19.0.dev0) for conversion and processing.

SHA256SUMS

@@ -0,0 +1,35 @@
+8759bdf7c77efc7df7723f64856a593c8943b71ee38baf2a88771fbaf78438f9  1_Pooling/config.json
+cdb09dfca347a56aa2d691744e38d5ad3c7cbc2834e7181272b9a15328b82524  2_Normalize/config.json
+bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a  LICENSE
+85ba8daf1bf1acd0a5cc239ff106203ace9c0c3717a9d8690dbb28bcb022d11d  NOTICE
+8926ecfb31c048bca42fd7c93bb64e92e8a7af767c4ceeaf1531b6b3b8e06761  README.md
+9fc2f9fa2e72d89965809df5e8e74c541dd297acc781b156e156d7bdf545ab38  RELATED_MODEL_RESEARCH.md
+46a0001714f1f99a5a38d5fc8b12838c60f606f54a10cbaf64315c927a8853ba  SOURCE.json
+b677a0ee2818c36f091b65fd3d9b4baea9f04d53b02b49cae916c74bf9682037  UPSTREAM_README.md
+0db0c4704bd2b44e73dbf6544e8cca29a2b957e4666bd9bbe459442b4947303a  audio.aimodel/main.hash
+d34c4312378bfe6a41f5b12091f4141fa8717889c82a377314be573969b095a3  audio.aimodel/main.mlirb
+773527bec44748e8f64fedbb4a7488ba6c583fc4b7c0f3d650212923ad9b3199  audio.aimodel/metadata.json
+4b852efc0b9960283e735363331e6f325b33bc74bdbaa076f595bc4e9b94d85e  chat_template.jinja
+b8f1e9931b57fbc054acdb445c41765d55b0074c58d145fa82839941ad1b5bb3  config.json
+031e56a498d33c349ab489a21885bcfe25b4fcba841149dc99e1e90d4a7c28f5  config_sentence_transformers.json
+2a731c4dd36cac9b96187b144e58c4396f2e38dc6245dfeba78f4b6e177e6356  convert_gemma2.py
+fc94d0291f4291e2d0acf22687843ba1e7ce2df91d03af900bc4aaeffdd09f6f  coreai_embeddinggemma2.py
+0dda0cd4eb3a160e81fe3eccc8466de577a5ea553e8eac2b2f6cf6ddbbc78bad  embedding.aimodel/main.hash
+b14af3c45b7d3749b575a76a654843a6b1a843b83b317d1e8d3a2355a8809f9c  embedding.aimodel/main.mlirb
+80eb34ddec601b8071d7f1cb4578e26bc98575f365fd79cc9e5080a921104bde  embedding.aimodel/metadata.json
+098a1496be00f9c5351c798b7545ca58bec296d38643ef56a940054c65f3536a  manifest.json
+3d02572a0455b832de67fb8e63a54981bc7e8b46e337c95e917bd8122a533bfd  modules.json
+ea2ae257e901064abdd98dceb19f2b0da06af600bed15e0f99f5c85c37ee9d78  preprocessor_config.json
+168f6a08522f3ce5dea596d94d003af2fd691742d4f41fe1f9d8cce76bfbf69c  processor_config.json
+dd0e8558adde3d71f5f8f69c169dce97ff7a86b9a8eddde0d921b8479440fb1a  requirements.txt
+b1bcd9f2dce3ae863b359e87d0710b5dbc3314a59ecb4e2f97c7778fc8e4b228  sentence_bert_config.json
+c6de103dfdd0e4d7c523210c4a90e4643f3efcc871f85c6abeea0c729f156760  smoke.json
+a13711b584f9ea10a5f78836b31af09129481d74af6665dee62bf906b51ac347  smoke_gemma2.py
+4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4  tokenizer.json
+e594c8a90eb08d8bda498ff4747977dc827ae0c3c56b5c0d41a605a22d02ef03  tokenizer.model
+17bd5d6e9364ca49a534e1502076593317c298d4a663623091ed45388f004874  tokenizer_config.json
+548a313b4610350ac2f7cd2d43742f6d242ad0e381600fa12e2558839fae0a36  validate_gemma2.py
+89fd15b469eabd478a046f10fad927f5d7a7678523f5d0efb1f438ff8840b699  validation.json
+e2249be21920798ddd44486045f52ed7d4a89d0321c96499e9d0baefa252b306  vision.aimodel/main.hash
+cabf97de2b23a35cecea87c802f4f9f12d0b38126d43af539e932ddf6935f978  vision.aimodel/main.mlirb
+aedfcb0574f0ab8e395f4e19a33865551e942229945f966eb106d7191ea63f57  vision.aimodel/metadata.json

SOURCE.json

@@ -0,0 +1,19 @@
+{
+  "repo": "google/embeddinggemma-2",
+  "revision": "914f7f89142e33e77833254d9c9b90c3cef7303b",
+  "license": "apache-2.0",
+  "verified_source_weights": {
+    "model.safetensors": {
+      "sha256": "197a32965d4b1105faf060417baa899e193fb73cd401f42ec9295234d5553d79",
+      "size": 1488915288
+    },
+    "tokenizer.json": {
+      "sha256": "4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4",
+      "size": 32170510
+    },
+    "tokenizer.model": {
+      "sha256": "e594c8a90eb08d8bda498ff4747977dc827ae0c3c56b5c0d41a605a22d02ef03",
+      "size": 4689013
+    }
+  }
+}

UPSTREAM_README.md

@@ -0,0 +1,335 @@
+---
+license: apache-2.0
+pipeline_tag: feature-extraction
+tags:
+- embedding
+- feature-extraction
+- sentence-transformers
+- multimodal-embedding
+- multimodal
+- vision
+- audio
+- video
+- image-feature-extraction
+- audio-feature-extraction
+- video-feature-extraction
+- sentence-similarity
+language:
+- multilingual
+library_name: transformers
+---
+
+<div align="center">
+  <img src=https://ai.google.dev/gemma/images/embeddinggemma2_banner.png>
+</div>
+
+<p align="center">
+    <a href="https://huggingface.co/google/embeddinggemma-2" target="_blank">Hugging Face</a> |
+    <a href="https://github.com/google-gemma" target="_blank">GitHub</a> |
+    <a href="https://blog.google/innovation-and-ai/technology/developers-tools/embeddinggemma-2" target="_blank">Launch Blog</a> |
+    <a href="https://ai.google.dev/gemma/docs/embeddinggemma" target="_blank">Documentation</a> |
+    <br>
+    <b>License</b>: <a href="https://ai.google.dev/gemma/docs/gemma_4_license" target="_blank">Apache 2.0</a> | <b>Authors</b>: <a href="https://deepmind.google/models/gemma/" target="_blank">Google DeepMind</a>
+</p>
+
+**EmbeddingGemma 2** is an open multimodal embedding model built by Google DeepMind which maps text (incl. code), images, video, and audio inputs—and combinations thereof—into a single, unified 768-dimensional vector space. The model has 740M total parameters, combining a 270M parameter text model with modular vision (170M) and audio (300M) encoders.
+
+Designed to run on consumer hardware such as mobile devices and laptops, EmbeddingGemma 2 delivers low-latency semantic representations for on-device applications, like search, retrieval-augmented generation (RAG), classification, and clustering.
+
+EmbeddingGemma 2 builds upon the architectural and capability advancements of Gemma 4, offering several core features:&nbsp;
+
+* **Native multimodality:** Native multimodality: Unifies 4 modalities (text, images, video, and audio) in a single shared 768-dimensional embedding space.
+* **Multilinguality and code:** EmbeddingGemma 2 understands 100+ languages, and achieves a \~14% improvement on code tasks relative to its predecessor.&nbsp;  
+* **Flexible footprint:** Combines a 270M parameter text backbone (130M transformer \+ 140M embedder) with selectively loadable vision (170M) and audio (300M) encoders, allowing developers to load only the modalities required for their use case.  
+* **Matryoshka Representation Learning (MRL):** Native support for truncated embeddings across 128d, 256d, 512d, and 768d, enabling up to a **6x reduction** in vector storage costs with minimal impact on quality.  
+* **Context length:** 8K token context window, capable of processing minutes of audio or video.
+* **Task-steered representations:** Uses lightweight text instruction prefixes to optimize embeddings for different tasks (search, classification, clustering, semantic similarity, etc.).
+
+### Model Overview
+
+| Parameters | Total | 740M |
+| :---- | :---- | :---- |
+|  | **Backbone** | 130M |
+|  | **Embedder** | 140M |
+|  | **Modality Encoders** | *Vision:* 170M  *Audio:* 300M |
+| **Architecture** | **Layers** | 24 |
+|  | **Model Dimension** | 512 |
+|  | **Hidden Dimension** | 2048 |
+|  | **Sliding Window** | 1024 tokens |
+|  | **Vocabulary Size** | 262,144 |
+|  | **\# Heads** | 4 |
+|  | **\# KV-Heads (Local/Global)** | 2/1 |
+|  | **Local:Global** | 5:1 |
+|  | **Attention** | GQA/MQA |
+|  | **Activation** | Gated FFN with GELU |
+|  | **Pooling** | Mean Pooling |
+|  | **Projection Layer** | 512→768 |
+| **Input/Output** | **Supported Modalities** | Text, Images, Video, Audio |
+|  | **Context Window** | 8,192 tokens |
+|  | **Native Output Dimension** | 768 |
+|  | **MRL Truncation Dimensions** | 128, 256, 512 |
+
+## **Benchmark Results**
+
+EmbeddingGemma 2 was evaluated across text, code, vision, visual document, video, and audio embedding benchmarks. All results reported below use the full-precision checkpoint.
+
+### Overall Evaluation Results (768d)
+
+| Modality | Benchmark | Metric | EmbeddingGemma 2 | EmbeddingGemma 1 |
+| :---- | :---- | :---- | :---- | :---- |
+| *Text* | Massive Text Embedding Benchmark (MTEB, multilingual, v2) | Mean(Task), Multiple | 61.36 | 61.15 |
+|  | Massive Text Embedding Benchmark (MTEB, code, v1) | Mean(Task), NDCG@10 | 78.68 | 68.76 |
+| *Image* | Massive Image Embedding Benchmark (MIEB, lite) | Mean(TaskType), Multiple | 64.64 | \- |
+|  | Massive Multimodal Embedding Benchmark (MMEB v2 \- Image) | Mean(Task), Hit@1 | 57.28 | \- |
+|  | Massive Multimodal Embedding Benchmark (MMEB v2 \- VisDoc) | Mean(Task), NDCG@5 | 67.84 | \- |
+| *Video* | Massive Multimodal Embedding Benchmark (MMEB v2 \- Video) | Mean(Task), Hit@1 | 50.67 | \- |
+| *Audio* | Massive Sound Embedding Benchmark (MSEB, Retrieval) | Mean(Task), MRR@10 | 69.54 | \- |
+|  | Massive Audio Embedding Benchmark (MAEB) *Hugging Face*&nbsp; | Mean(Task), Multiple | 49.39 | \- |
+
+### Evaluation Results with Vector Truncation
+
+With MRL, EmbeddingGemma 2 representations can be truncated below the native 768d to 128d, 256d, and 512d representations and re-normalized.
+With this, model users can reduce storage requirements, with minimal quality impact down to 256d. 128d is best suited to text-only workloads.
+
+| Output Dimension | Compression Ratio | MTEB *(multilingual, v2) Mean(Task)* | MTEB  *(eng, v2) Mean(Task)* | MTEB *(code, v1) Mean(Task)* | MIEB *(lite) Mean(TaskType)* | MMEB (v2) *Overall* | MSEB  *(Retrieval) Mean(Task)* | MAEB   *Mean(Task)* |
+| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
+| 768d (Full) | 1:1 | 61.36 | 68.46 | 78.68 | 64.64 | 59.01 | 69.54 | 49.39 |
+| 512d | 1:1.5 | 61.17 | 68.41 | 77.24 | 64.32 | 58.38 | 69.18 | 49.21 |
+| 256d | 1:3 | 60.41 | 67.78 | 76.18 | 63.13 | 56.24 | 66.76 | 48.91 |
+| 128d | 1:6 | 57.89 | 65.68 | 71.41 | 59.06 | 45.65 | 56.71 | 46.92 |
+
+## **Quick Start**
+
+Install the `sentence-transformers` library: 
+
+```bash
+pip install -U sentence-transformers transformers
+```
+
+Generate text embeddings:
+```python
+from sentence_transformers import SentenceTransformer
+
+model = SentenceTransformer("google/embeddinggemma-2")
+
+query = "What causes the northern lights?"
+document = "The northern lights are caused by charged particles from the sun."
+
+query_emb = model.encode(query, prompt_name="SearchQuery")
+doc_emb = model.encode(document, prompt_name="Document")
+print(model.similarity(query_emb, doc_emb))
+```
+
+
+## **Best Practices**
+
+For optimal embedding quality and runtime efficiency, follow these configurations and best practices:
+
+### 1. Task Instruction Prefixes
+
+EmbeddingGemma 2 is trained with short task instruction prefixes prepended to text inputs. Using the right prefix improves quality; omitting it still works but reduces precision. Prefixes apply to text only. Pass images, video, and audio without any prefix. 
+
+Documents with a real title should be formatted as `title: {title} | text: {content}`. Use `title: none` when no title is available.
+
+**Prefix Notation & Usage**
+
+We offer two types of task prefixes, depending on how embeddings are used in the task. There are two types of tasks:
+
+* **Asymmetric Tasks (e.g. retrieval):** Use a query prefix for queries and a document prefix for corpus items.  
+* **Symmetric Tasks (e.g. classification, similarity)**: Apply the same task prefix to all inputs being compared.
+
+| Use Case | Task Type | Prompt Name | Query Task Instruction | Document Task Instruction *(use `none` if no title)* |
+| :---- | :---- | :---- | :---- | :---- |
+| Web / document search | Asymmetric | SearchQuery | `task: search result \| query: {query}` | `title: {title} \| text: {content}` |
+| Question answering | Asymmetric | QuestionAnswering | `task: question answering \| query: {question}` | `title: {title} \| text: {passage}` |
+| Fact checking | Asymmetric | FactChecking | `task: fact checking \| query: {claim}` | `title: {title} \| text: {evidence}` |
+| Code search | Asymmetric | CodeRetrieval | `task: code retrieval \| query: {query}` | `title: {title or filename} \| text: {code}` |
+| Text classification | Symmetric | Classification | `task: classification \| query: {content}` | N/A |
+| Clustering | Symmetric | Clustering | `task: clustering \| query: {content}` | N/A |
+| Measuring similarity | Symmetric | SentenceSimilarity | `task: sentence similarity \| query: {content}` | N/A |
+
+**Please note:** `prompt_name="Document"` applies `title: none`; titled documents must still be formatted manually, like `model.encode(f"title: {title} | text: {document}")`
+
+### 2. Selective Encoder Loading
+
+The vision and audio encoders are independent components. To reduce memory consumption when deploying text-only or single-modality pipelines, disable unused modality encoders via SentenceTransformer’s `config_kwargs`:
+
+**Please note:** configuring EmbeddingGemma 2 to load with omission of some encoders differs among model libraries; please refer to the appropriate documentation for more information on this.
+
+| Active Modalities | `config_kwargs` | Effective Size |
+| :---- | :---- | :---- |
+| Text only | `{"vision_config": None, "audio_config": None}` | 270M |
+| Text and image | `{"audio_config": None}` | 440M |
+| Text and audio | `{"vision_config": None}` | 570M |
+| Full multimodal | `{}` | 740M |
+
+### 3. Matryoshka Dimension Truncation
+
+EmbeddingGemma 2 is trained with Matryoshka Representation Learning, so the 768-dimensional output vector can be shortened by keeping only its leading dimensions. The supported dimensions are 768, 512, 256, and 128\. Shorter vectors reduce storage and speed up similarity search at some cost to quality.
+
+At runtime, please adhere to the following guidelines:
+
+* **Re-normalize after truncating:** slicing a unit-length vector does not preserve unit length. The shortened vector must be L2-normalized before it is used for cosine similarity. Skipping this step degrades ranking quality silently—it produces plausible-looking scores rather than an error.  
+* **Queries and documents must share a dimension.** A 768-dimensional query cannot be scored against a 128-dimensional corpus.
+* 
+To avoid truncating and normalizing yourself, you should pass in the `truncate_dim` and `normalize_embeddings` fields when calling `model.encode()`:
+
+```python
+query_emb = model.encode(
+   query,
+   truncate_dim=128, # or 512, 256
+   normalize_embeddings=True,
+)
+```
+
+Model quality at each dimension is reported in the truncation table in the **Benchmark Results** section above. Quality is close to lossless down to 256 dimensions. 128 dimensions degrades multimodal quality substantially and should be validated against your own workload before adoption.
+
+### 4. Numerical Precision
+
+Run inference in `bfloat16` or `float32`. Do not use `float16`.
+
+EmbeddingGemma 2's activation range exceeds the dynamic range of `float16`. In `float16` the model returns NaN or silently degraded embeddings rather than raising an error, so the failure is easy to miss.
+
+`bfloat16` is safe, and carries the same 8-bit exponent as `float32`. This is the recommended default on hardware with native support, and halves memory use relative to `float32`. Use `float32` elsewhere, including on most CPUs.
+
+In SentenceTransformers, you can set this via `model_kwargs`, and determine it programmatically via `is_bf16_supported`:
+
+```python
+dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32
+
+model = SentenceTransformer("google/embeddinggemma-2", model_kwargs={"torch_dtype": dtype})
+```
+
+### 5. Multimodal Input
+
+#### Interleaving
+
+A single input may mix text with images, video, and audio, using a single shared 8,192-token context.
+
+The position of each media item within the sequence is marked in the text using placeholder tokens from the model's vocabulary:
+
+* `<|image|>` marks the position of image input  
+* `<|video|>` marks the position of video input  
+* `<|audio|>` marks the position of audio input&nbsp;
+
+For example, a product listing indexed for search might be encoded as:
+
+```python
+emb_interleaved = model.encode({
+    "text": "Waterproof running shoes. <|image|> Featuring a breathable mesh upper. <|image|> Grip test on wet rock: <|video|>",
+    "image": ["shoe.jpg", "mesh.jpg"],
+    "video": "demo.mp4",
+})
+```
+
+Each placeholder in text is filled from the corresponding key, in order. The call returns a single embedding that represents the text, images and video together. This embedding can be compared directly against any other EmbeddingGemma 2 embedding; for example, a text-only query such as “waterproof shoes for trail running”.
+
+#### Context Limits
+
+All modalities share a single 8,192-token context window. Each modality consumes that budget at a fixed rate:
+
+| Modality | Token Cost | Max Input |
+| :---- | :---- | :---- |
+| *Text* | 1 token per subword | 8,192 tokens |
+| *Image* | 280 tokens per image  *(default)* | ~29 images |
+| *Video* | 140 tokens per frame  *(default)* | ~58 frames |
+| *Audio* | 25 tokens per second | ~327 seconds |
+
+Note max input for images and video can be as high as ~114 images or frames when using a lower vision token budget (described below).
+
+The maximums above assume a single modality with no accompanying text. Interleaved inputs draw from the same budget, so mixing modalities reduces the amount of each that fits. Note also max input for images and video can be as high as \~114 images or frames when using a lower vision token budget (described below).
+
+#### Vision Token Budget
+
+Model users can forgo the default sequence length for input images / video frames to represent images with “soft token” amounts ranging from 70 to 1120\. Increasing the vision budget for input images trades latency and token count for quality.&nbsp;
+
+In general, higher input sequence lengths capture more information. Scaling up input sequence length means more expressive input representations; so, increasing input sequence length will improve fine-grained visual understanding, uplifting embedding quality and performance in downstream use cases.
+
+#### Input Sampling Defaults
+
+Audio and video inputs have the following default sampling rates:
+
+* By default, video is processed as sampled frames through the vision encoder at **1 frame per second;** FPS sampling rate is configurable.  
+* Audio should be supplied as **mono at 16 kHz.**
+
+## **Model Data**
+
+Our pre-training dataset is a large-scale, diverse collection of data encompassing a wide range of domains and modalities, which includes web documents, code, images, video, and audio, with a cutoff date of January 2025\. These include:
+
+* **Web Documents:** A diverse collection of web text ensures the model is exposed to a broad range of linguistic styles, topics, and vocabulary. The training dataset includes content in over 140 languages.  
+* **Code:** Exposing the model to code helps it to learn the syntax and patterns of programming languages, which improves its ability to understand code-related semantics.&nbsp;  
+* **Images:** A wide range of images enables the model to perform image analysis and visual data extraction tasks  
+* **Video:** Video sequences and clips covering diverse visual scenes, human actions, and temporal dynamics.  
+* **Audio:** Speech recordings across multiple languages, environmental sounds, and acoustic events.  
+* **Cross-modality Samples:** Paired text, image, video, and audio data to align representations across different modalities.
+
+### Data Processing
+
+Several data cleaning and filtering methods were applied to the training data:
+
+* **CSAM Filtering**: Rigorous CSAM (Child Sexual Abuse Material) filtering was applied at multiple stages in the data preparation process to ensure the exclusion of harmful and illegal content.  
+* **Sensitive Data Filtering**: As part of making Gemma pre-trained models safe and reliable, automated techniques were used to filter out certain personal information and other sensitive data from training sets.  
+* **Additional Methods**: Filtering based on content quality and safety in line with [our policies](https://ai.google/static/documents/ai-responsibility-update-published-february-2025.pdf).
+
+### **Ethics & Safety**
+
+EmbeddingGemma 2 is a pre-trained embedding model. Unlike generative models, it does not undergo post-training alignment, safety tuning, or output-level moderation. Safety mitigations during development were focused on pre-training data filtering to reduce exposure to harmful content and severe biases in the learned embedding space, in alignment with [Google's AI Principles](https://ai.google/principles/).
+
+Since embedding models produce representations rather than user-facing text, safety risks manifest downstream in how those representations are used.&nbsp;
+
+Developers and deployers are responsible for evaluating and implementing application-level safeguards, like retrieval filtering and fairness testing, appropriate to their specific production use case.&nbsp;
+
+Deployments must adhere to the [Gemma Prohibited Use Policy](https://ai.google.dev/gemma/prohibited_use_policy).
+
+## **Usage and Limitations**
+
+EmbeddingGemma 2 has certain limitations that users should be aware of.
+
+### Intended Usage
+
+EmbeddingGemma 2 generates embeddings from input content, which can be used for a number of downstream applications.
+
+The following list of potential uses is not comprehensive. The purpose of this list is to provide contextual information about the possible use-cases that the model creators considered as part of model training and development.
+
+* **Retrieval:** Embeddings for semantic search across text, code, images, video, and audio, such as document search, RAG over enterprise knowledge bases, code search from natural language queries, or spoken-query search over audio archives.  
+* **Classification:** Embeddings to classify inputs according to preset labels, like sentiment analysis, content moderation, audio event detection, or image categorization.  
+* **Clustering:** Embeddings to group inputs based on semantic similarity, like organizing document collections, clustering customer feedback by theme, or discovering near-duplicate content across modalities.  
+* **Semantic Similarity:** Embeddings to measure pairwise similarity between inputs, such as recommendation systems, duplicate detection, paraphrase identification, or cross-lingual content alignment.  
+* **Fact Verification:** Embeddings for retrieving evidence documents given a claim, like automated fact-checking systems or source attribution pipelines.
+
+### Limitations
+
+* **Training Data:** The quality and diversity of the training data significantly influence the model's capabilities. Biases or gaps in the training data can lead to limitations in the model's responses.  
+  * The scope of the training dataset determines the subject areas the model can handle effectively.  
+  * For example, while EmbeddingGemma 2 supports 100+ languages, the model may not exhibit equal performance across languages.&nbsp;  
+* **Context and Task Complexity:** Models perform well on tasks that can be framed with clear prompts and instructions.&nbsp;  
+  * Open-ended or highly complex tasks might be challenging.  
+  * A model's performance can be influenced by the amount of context provided (longer context generally leads to better outputs, up to a certain point).  
+* **Language Ambiguity and Nuance:** Natural language is inherently complex. Models might struggle to grasp subtle nuances, sarcasm, or figurative language.  
+* **Task Instruction Usage:** For text tasks, omitting the recommended task prefix may lead to sub-optimal embedding quality.
+
+### Ethical Considerations and Risks
+
+In creating an open embedding model, we have carefully considered the following:
+
+* **Bias and Fairness**  
+  * Models trained on large-scale, real-world text and image data can reflect socio-cultural biases embedded in the training material.&nbsp;  
+  * Training data used for EmbeddingGemma 2 underwent safety filtering to mitigate the risk of these biases.  
+* **Misinformation and Misuse**  
+  * Embedding representations can be misused to retrieve, classify, or otherwise organize embedded content in false, misleading or harmful ways.&nbsp;  
+  * Guidelines are provided for responsible use with the model, see the [Responsible Generative AI Toolkit](https://ai.google.dev/responsible).
+* **Transparency and Accountability**
+  * This model card summarizes details on the model's architecture, capabilities, limitations, and evaluation processes.
+  * A responsibly developed open model offers the opportunity to share innovation by making Vision-Language Model (VLM) technology accessible to developers and researchers across the AI ecosystem. 
+
+### Risks Identified and Mitigations
+
+* **Misuse for malicious purposes:** Technical limitations and developer and end-user education can help mitigate against malicious applications of embedding models. Educational resources and reporting mechanisms for users to flag misuse are provided.  
+* **Privacy violations:** Models were trained on data filtered for removal of certain personal information and other sensitive data. Developers are encouraged to adhere to privacy regulations with privacy-preserving techniques.  
+* **Perpetuation of biases:** It's encouraged to perform continuous monitoring (using evaluation metrics, human review) and the exploration of de-biasing techniques during model training, fine-tuning, and other use cases.
+
+### Benefits
+
+EmbeddingGemma 2 is among the strongest multimodal embedding models under 1B parameters. We’re excited to see how developers will use and adapt this model for on-device or edge AI applications.&nbsp;
+
+The model is designed from the ground up for responsible AI development, like other Gemma-family models.

audio.aimodel/main.hash

@@ -0,0 +1 @@
+�LC7��jA�� ���qx��*7s�W9i���

audio.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d34c4312378bfe6a41f5b12091f4141fa8717889c82a377314be573969b095a3
+size 1223634908

audio.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "producer" : "coreai-core 1.0.0b3",
+  "assetVersion" : "2.0",
+  "creationDate" : "20261011T130713Z"
+}

chat_template.jinja

@@ -0,0 +1 @@
+{%- for msg in messages if msg.get('role') == 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- for item in msg['content'] if item.get('type') == 'text' -%}{{ item['text'] }}{%- endfor -%}{%- endif -%}{%- endfor -%}{%- for msg in messages if msg.get('role') != 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}{%- for item in msg['content'] -%}{%- if item.get('type') == 'text' -%}{{ item['text'] }}{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}<|image|>{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}<|video|>{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}<|audio|>{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}

config.json

@@ -0,0 +1,177 @@
+{
+  "architectures": [
+    "EmbeddingGemma2Model"
+  ],
+  "audio_config": {
+    "_name_or_path": "",
+    "architectures": null,
+    "attention_chunk_size": 12,
+    "attention_context_left": 13,
+    "attention_context_right": 0,
+    "attention_invalid_logits_value": -1000000000.0,
+    "attention_logit_cap": 50.0,
+    "chunk_size_feed_forward": 0,
+    "conv_kernel_size": 5,
+    "dtype": "bfloat16",
+    "gradient_clipping": 10000000000.0,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "model_type": "gemma4_audio",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 12,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_proj_dims": 1536,
+    "problem_type": null,
+    "residual_weight": 0.5,
+    "return_dict": true,
+    "rms_norm_eps": 1e-06,
+    "subsampling_conv_channels": [
+      128,
+      32
+    ],
+    "use_clipped_linears": true
+  },
+  "audio_token_id": 258881,
+  "boa_token_id": 256000,
+  "boi_token_id": 255999,
+  "dtype": "bfloat16",
+  "eoa_token_index": 258883,
+  "eoi_token_id": 258882,
+  "image_token_id": 258880,
+  "initializer_range": 0.02,
+  "model_type": "embedding_gemma2",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 2,
+    "dtype": "bfloat16",
+    "embedding_dim": 768,
+    "eos_token_id": 1,
+    "head_dim": 256,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 512,
+    "hidden_size_per_layer_input": 512,
+    "initializer_range": 0.02,
+    "intermediate_size": 2048,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 262144,
+    "model_type": "embedding_gemma2_text",
+    "num_attention_heads": 4,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "pad_token_id": 0,
+    "per_layer_config": {
+      "05": {
+        "head_dim": 512,
+        "num_key_value_heads": 1
+      },
+      "11": {
+        "head_dim": 512,
+        "num_key_value_heads": 1
+      },
+      "17": {
+        "head_dim": 512,
+        "num_key_value_heads": 1
+      },
+      "23": {
+        "head_dim": 512,
+        "num_key_value_heads": 1
+      }
+    },
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "full_attention": {
+        "rope_theta": 1000000.0,
+        "rope_type": "default"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
+    "sliding_window": 512,
+    "vocab_size": 262144
+  },
+  "transformers_version": "5.18.0.dev0",
+  "video_token_id": 258884,
+  "vision_config": {
+    "_name_or_path": "",
+    "architectures": null,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "chunk_size_feed_forward": 0,
+    "default_output_length": 280,
+    "dtype": "bfloat16",
+    "global_head_dim": 64,
+    "head_dim": 64,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "max_position_embeddings": 131072,
+    "model_type": "gemma4_vision",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 16,
+    "num_key_value_heads": 12,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "position_embedding_size": 10240,
+    "problem_type": null,
+    "return_dict": true,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 100.0,
+      "rope_type": "axial"
+    },
+    "standardize": false,
+    "use_clipped_linears": false
+  },
+  "vision_soft_tokens_per_image": 280
+}

config_sentence_transformers.json

@@ -0,0 +1,38 @@
+{
+  "__version__": {
+    "pytorch": "2.14.0+cu130",
+    "sentence_transformers": "6.1.0",
+    "transformers": "5.18.0.dev0"
+  },
+  "default_prompt_name": null,
+  "model_type": "SentenceTransformer",
+  "prompts": {
+    "BitextMining": "task: search result | query: ",
+    "Classification": "task: classification | query: ",
+    "Clustering": "task: clustering | query: ",
+    "CodeRetrieval": "task: code retrieval | query: ",
+    "Document": "title: none | text: ",
+    "FactChecking": "task: fact checking | query: ",
+    "InstructionRetrieval": "task: code retrieval | query: ",
+    "MultilabelClassification": "task: classification | query: ",
+    "PairClassification": "task: sentence similarity | query: ",
+    "QuestionAnswering": "task: question answering | query: ",
+    "Reranking": "task: search result | query: ",
+    "Retrieval": "task: search result | query: ",
+    "Retrieval-document": "title: none | text: ",
+    "Retrieval-query": "task: search result | query: ",
+    "STS": "task: sentence similarity | query: ",
+    "SearchQuery": "task: search result | query: ",
+    "SentenceSimilarity": "task: sentence similarity | query: ",
+    "Summarization": "task: sentence similarity | query: ",
+    "document": "title: none | text: ",
+    "query": "task: search result | query: "
+  },
+  "requirements": {
+    "sentence-transformers": {
+      "reason": "Older versions ignore the order of multimodal dict inputs and cannot put several images or texts into a single embedding.",
+      "specifier": ">=6.1.0"
+    }
+  },
+  "similarity_fn_name": "cosine"
+}

convert_gemma2.py

@@ -0,0 +1,148 @@
+import json
+from pathlib import Path
+
+import torch
+import torch.nn.functional as F
+from coreai_torch import TorchConverter, get_decomp_table
+from transformers import AutoModel, AutoProcessor
+
+
+def unfold(x, dimension, size, step):
+    dimension %= x.ndim
+    return torch.stack(
+        [
+            x.narrow(dimension, i, size)
+            for i in range(0, x.shape[dimension] - size + 1, step)
+        ],
+        dim=dimension,
+    ).movedim(dimension + 1, -1)
+
+
+def save(model, sample, path):
+    with torch.no_grad():
+        ep = torch.export.export(model, sample, strict=False)
+        table = get_decomp_table()
+        table[torch.ops.aten.unfold.default] = unfold
+        ep = ep.run_decompositions(table)
+    print("Converting", path, flush=True)
+    TorchConverter().add_exported_program(ep).to_coreai().save_asset(path)
+    return list(ep.graph_signature.user_outputs)
+
+
+class Text(torch.nn.Module):
+    def __init__(self, m):
+        super().__init__()
+        self.m = m.language_model
+
+    def forward(self, input_ids, attention_mask, media_embeddings, media_mask):
+        emb = self.m.embed_tokens(input_ids)
+        emb = torch.where(media_mask, media_embeddings.to(emb.dtype), emb)
+        n = input_ids.shape[1]
+        pos = torch.arange(n, device=input_ids.device).unsqueeze(0)
+        mask = attention_mask[:, None, None, :].bool().expand(1, 1, n, n)
+        sliding = (
+            pos[:, :, None] - pos[:, None, :]
+        ).abs() <= self.m.config.sliding_window
+        hidden = self.m(
+            inputs_embeds=emb,
+            position_ids=pos,
+            attention_mask={
+                "full_attention": mask,
+                "sliding_attention": mask & sliding[:, None],
+            },
+        ).last_hidden_state
+        weights = attention_mask[:, :, None].float()
+        pooled = (hidden.float() * weights).sum(1) / weights.sum(1).clamp(min=1)
+        return F.normalize(pooled, dim=-1)
+
+
+class Vision(torch.nn.Module):
+    def __init__(self, m):
+        super().__init__()
+        self.v = m.vision_tower
+        self.project = m.embed_vision
+
+    def forward(self, pixel_values, pixel_position_ids):
+        v = self.v
+        pad = (pixel_position_ids == -1).all(-1)
+        emb = v.patch_embedder(pixel_values, pixel_position_ids, pad)
+        hidden = v.encoder(
+            inputs_embeds=emb,
+            attention_mask=~pad,
+            pixel_position_ids=pixel_position_ids,
+        ).last_hidden_state
+        hidden, mask = v.pooler(
+            hidden, pixel_position_ids, pad, pixel_values.shape[1] // 9
+        )
+        return self.project(hidden.to(emb.dtype)), mask
+
+
+class Audio(torch.nn.Module):
+    def __init__(self, m):
+        super().__init__()
+        self.a = m.audio_tower
+        self.project = m.embed_audio
+
+    def forward(self, input_features, input_features_mask):
+        out = self.a(input_features, attention_mask=input_features_mask)
+        return self.project(out.last_hidden_state), out.attention_mask
+
+
+if __name__ == "__main__":
+    torch.set_num_threads(4)
+    root = Path("bundles/embeddinggemma-2-CoreAI")
+    root.mkdir(parents=True, exist_ok=True)
+    m = AutoModel.from_pretrained(
+        "upstream/embeddinggemma-2",
+        dtype=torch.float32,
+        attn_implementation={
+            "language_model": "sdpa",
+            "vision_tower": "sdpa",
+            "audio_tower": "eager",
+        },
+    ).eval()
+    print("Loaded", flush=True)
+    manifest = {
+        "source": "google/embeddinggemma-2",
+        "revision": "914f7f89142e33e77833254d9c9b90c3cef7303b",
+        "precision": "float32",
+        "sequence_length": 1024,
+        "transformers_revision": "536ecc007387a50e77603bb5d92100e9b07514cc",
+    }
+    sample = (
+        torch.ones(1, 1024, dtype=torch.int32),
+        torch.ones(1, 1024, dtype=torch.int32),
+        torch.zeros(1, 1024, 512),
+        torch.zeros(1, 1024, 1, dtype=torch.bool),
+    )
+    manifest["text_outputs"] = save(Text(m).eval(), sample, root / "embedding.aimodel")
+    (root / "manifest.json").write_text(json.dumps(manifest, indent=2))
+    p = AutoProcessor.from_pretrained("upstream/embeddinggemma-2")
+    from PIL import Image
+
+    e = p(images=Image.new("RGB", (224, 224)), return_tensors="pt")
+    print("Image shapes", [(k, tuple(v.shape)) for k, v in e.items()], flush=True)
+    sample = (e["pixel_values"].float(), e["image_position_ids"].int())
+    manifest["vision_outputs"] = save(Vision(m).eval(), sample, root / "vision.aimodel")
+    manifest["vision_shape"] = list(sample[0].shape)
+    (root / "manifest.json").write_text(json.dumps(manifest, indent=2))
+    import numpy as np
+
+    e = p(
+        audio=np.sin(np.arange(16000) * 0.1).astype(np.float32),
+        sampling_rate=16000,
+        return_tensors="pt",
+    )
+    print("Audio shapes", [(k, tuple(v.shape)) for k, v in e.items()], flush=True)
+    sample = (
+        F.pad(
+            e["input_features"].float(), (0, 0, 0, 1120 - e["input_features"].shape[1])
+        ),
+        F.pad(
+            e["input_features_mask"].bool(),
+            (0, 1120 - e["input_features_mask"].shape[1]),
+        ),
+    )
+    manifest["audio_outputs"] = save(Audio(m).eval(), sample, root / "audio.aimodel")
+    manifest["audio_shape"] = list(sample[0].shape)
+    (root / "manifest.json").write_text(json.dumps(manifest, indent=2))

coreai_embeddinggemma2.py

@@ -0,0 +1,163 @@
+"""EmbeddingGemma 2 adapter. Neural computation runs exclusively in Core AI."""
+
+import json
+from pathlib import Path
+
+import numpy as np
+from coreai.runtime import AIModel, NDArray, SpecializationOptions
+from transformers import AutoProcessor
+
+
+class EmbeddingGemma2CoreAI:
+    @classmethod
+    async def load(cls, bundle):
+        self = cls()
+        self.root = Path(bundle)
+        self.manifest = json.loads((self.root / "manifest.json").read_text())
+        self.processor = AutoProcessor.from_pretrained(self.root)
+        self.prompts = json.loads(
+            (self.root / "config_sentence_transformers.json").read_text()
+        )["prompts"]
+        self.models = {}
+        self.functions = {}
+        for name, asset in [
+            ("text", "embedding"),
+            ("vision", "vision"),
+            ("audio", "audio"),
+        ]:
+            self.models[name] = await AIModel.load(
+                self.root / f"{asset}.aimodel", SpecializationOptions.default()
+            )
+            self.functions[name] = self.models[name].load_function("main")
+        return self
+
+    async def call(self, name, inputs):
+        result = await self.functions[name](
+            {k: NDArray(np.ascontiguousarray(v)) for k, v in inputs.items()}
+        )
+        return [result[k].numpy() for k in self.manifest[f"{name}_outputs"]]
+
+    async def encode(
+        self,
+        text=None,
+        *,
+        images=None,
+        videos=None,
+        audio=None,
+        sampling_rate=16000,
+        prompt_name=None,
+        truncate_dim=768,
+    ):
+        if not 1 <= truncate_dim <= 768:
+            raise ValueError("truncate_dim must be between 1 and 768")
+        if prompt_name:
+            if images is not None or videos is not None or audio is not None:
+                raise ValueError("Task prompts apply to text-only inputs")
+            text = self.prompts[prompt_name] + (text or "")
+        kwargs = {"return_tensors": "np"}
+        if text is not None:
+            kwargs["text"] = text
+        if images is not None:
+            kwargs["images"] = images
+        if videos is not None:
+            if not isinstance(videos, np.ndarray) or videos.ndim != 4:
+                raise ValueError(
+                    "Video must be an already sampled NumPy [frames,height,width,channels] array"
+                )
+            kwargs["videos"] = videos
+            kwargs["videos_kwargs"] = {"do_sample_frames": False}
+        if audio is not None:
+            samples = np.asarray(audio)
+            if sampling_rate != 16000:
+                raise ValueError("Audio must be sampled at 16000 Hz")
+            if samples.ndim != 1 or len(samples) > 179200:
+                raise ValueError("One mono clip of at most 11.2 seconds is supported")
+            kwargs.update(audio=samples, sampling_rate=sampling_rate)
+        encoded = self.processor(**kwargs)
+        return await self.encode_processed(encoded, truncate_dim=truncate_dim)
+
+    async def encode_processed(self, e, truncate_dim=768):
+        ids = np.asarray(e["input_ids"], dtype=np.int32)
+        if ids.shape[0] != 1:
+            raise ValueError(
+                "Batch size must be one; call encode separately per example"
+            )
+        n = ids.shape[1]
+        cap = self.manifest["sequence_length"]
+        if n > cap:
+            raise ValueError(
+                f"Input needs {n} tokens, capacity {cap}; input was not truncated"
+            )
+        media = np.zeros((1, cap, 512), np.float32)
+        media_mask = np.zeros((1, cap, 1), bool)
+        p = self.processor
+        for pixels_key, pos_key, token in [
+            ("pixel_values", "image_position_ids", p.image_token_id),
+            ("pixel_values_videos", "video_position_ids", p.video_token_id),
+        ]:
+            if pixels_key not in e:
+                continue
+            features = []
+            for pixels, positions in zip(e[pixels_key], e[pos_key]):
+                count = len(pixels)
+                capacity = self.manifest["vision_shape"][1]
+                if count > capacity:
+                    raise ValueError("Vision patch capacity exceeded")
+                pix = np.pad(
+                    np.asarray(pixels, dtype=np.float32),
+                    ((0, capacity - count), (0, 0)),
+                )
+                pos = np.pad(
+                    np.asarray(positions, dtype=np.int32),
+                    ((0, capacity - count), (0, 0)),
+                    constant_values=-1,
+                )
+                values, valid = await self.call(
+                    "vision",
+                    {"pixel_values": pix[None], "pixel_position_ids": pos[None]},
+                )
+                features.append(values[valid.astype(bool)])
+            features = np.concatenate(features, axis=0)
+            slots = np.flatnonzero(ids[0] == token)
+            if len(slots) != len(features):
+                raise ValueError("Visual placeholder count differs from encoder output")
+            media[0, slots] = features
+            media_mask[0, slots] = True
+        if "input_features" in e:
+            features = []
+            for mel, mask in zip(e["input_features"], e["input_features_mask"]):
+                count = len(mel)
+                capacity = self.manifest["audio_shape"][1]
+                if count > capacity:
+                    raise ValueError("Audio feature capacity exceeded")
+                mel = np.pad(
+                    np.asarray(mel, dtype=np.float32), ((0, capacity - count), (0, 0))
+                )
+                mask = np.pad(np.asarray(mask, dtype=bool), (0, capacity - count))
+                values, valid = await self.call(
+                    "audio",
+                    {"input_features": mel[None], "input_features_mask": mask[None]},
+                )
+                features.append(values[valid.astype(bool)])
+            features = np.concatenate(features, axis=0)
+            slots = np.flatnonzero(ids[0] == p.audio_token_id)
+            if len(slots) != len(features):
+                raise ValueError("Audio placeholder count differs from encoder output")
+            media[0, slots] = features
+            media_mask[0, slots] = True
+        padded = np.pad(ids, ((0, 0), (0, cap - n)))
+        mask = np.pad(
+            np.asarray(e["attention_mask"], dtype=np.int32), ((0, 0), (0, cap - n))
+        )
+        value = (
+            await self.call(
+                "text",
+                {
+                    "input_ids": padded,
+                    "attention_mask": mask,
+                    "media_embeddings": media,
+                    "media_mask": media_mask,
+                },
+            )
+        )[0][0, :truncate_dim]
+        return value / np.linalg.norm(value)

embedding.aimodel/main.hash

@@ -0,0 +1 @@
+�J��[}7I�u�jeHC���C�;1}�:#U����

embedding.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b14af3c45b7d3749b575a76a654843a6b1a843b83b317d1e8d3a2355a8809f9c
+size 1108810117

embedding.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "creationDate" : "20261011T130702Z",
+  "assetVersion" : "2.0",
+  "producer" : "coreai-core 1.0.0b3"
+}

manifest.json

@@ -0,0 +1,28 @@
+{
+  "source": "google/embeddinggemma-2",
+  "revision": "914f7f89142e33e77833254d9c9b90c3cef7303b",
+  "precision": "float32",
+  "sequence_length": 1024,
+  "transformers_revision": "536ecc007387a50e77603bb5d92100e9b07514cc",
+  "text_outputs": [
+    "div_1"
+  ],
+  "vision_outputs": [
+    "view_294",
+    "logical_not_8"
+  ],
+  "vision_shape": [
+    1,
+    2520,
+    768
+  ],
+  "audio_outputs": [
+    "view_542",
+    "slice_2"
+  ],
+  "audio_shape": [
+    1,
+    1120,
+    128
+  ]
+}

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.base.modules.transformer.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.sentence_transformer.modules.pooling.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.base.modules.normalize.Normalize"
+  }
+]

preprocessor_config.json

@@ -0,0 +1,21 @@
+{
+  "dither": 0.0,
+  "feature_extractor_type": "Gemma4AudioFeatureExtractor",
+  "feature_size": 128,
+  "fft_length": 512,
+  "fft_overdrive": false,
+  "frame_length": 320,
+  "hop_length": 160,
+  "input_scale_factor": 1.0,
+  "max_frequency": 8000.0,
+  "mel_floor": 0.001,
+  "min_frequency": 0.0,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "per_bin_mean": null,
+  "per_bin_stddev": null,
+  "preemphasis": 0.0,
+  "preemphasis_htk_flavor": true,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

processor_config.json

@@ -0,0 +1,78 @@
+{
+  "audio_ms_per_token": 40,
+  "audio_seq_length": 280,
+  "feature_extractor": {
+    "dither": 0.0,
+    "feature_extractor_type": "Gemma4AudioFeatureExtractor",
+    "feature_size": 128,
+    "fft_length": 512,
+    "fft_overdrive": false,
+    "frame_length": 320,
+    "hop_length": 160,
+    "input_scale_factor": 1.0,
+    "max_frequency": 8000.0,
+    "mel_floor": 0.001,
+    "min_frequency": 0.0,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "per_bin_mean": null,
+    "per_bin_stddev": null,
+    "preemphasis": 0.0,
+    "preemphasis_htk_flavor": true,
+    "return_attention_mask": true,
+    "sampling_rate": 16000
+  },
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": false,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.0,
+      0.0,
+      0.0
+    ],
+    "image_processor_type": "Gemma4ImageProcessor",
+    "image_seq_length": 280,
+    "image_std": [
+      1.0,
+      1.0,
+      1.0
+    ],
+    "max_soft_tokens": 280,
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098
+  },
+  "image_seq_length": 280,
+  "processor_class": "EmbeddingGemma2Processor",
+  "video_processor": {
+    "add_timestamps": false,
+    "do_convert_rgb": true,
+    "do_normalize": false,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 1,
+    "image_mean": [
+      0.0,
+      0.0,
+      0.0
+    ],
+    "image_std": [
+      1.0,
+      1.0,
+      1.0
+    ],
+    "max_frames": 32,
+    "max_soft_tokens": 140,
+    "overflow_strategy": "uniform",
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "video_processor_type": "EmbeddingGemma2VideoProcessor"
+  }
+}

requirements.txt

@@ -0,0 +1,9 @@
+torch==2.14.0
+coreai-torch==0.4.3
+coreai-core==1.0.0b3
+transformers @ git+https://github.com/huggingface/transformers.git@536ecc007387a50e77603bb5d92100e9b07514cc
+numpy==2.4.6
+Pillow==12.3.0
+torchvision==0.29.1
+safetensors==0.8.0
+huggingface-hub==1.33.0

sentence_bert_config.json

@@ -0,0 +1,27 @@
+{
+    "transformer_task": "feature-extraction",
+    "modality_config": {
+        "text": {
+            "method": "forward",
+            "method_output_name": "last_hidden_state"
+        },
+        "image": {
+            "method": "forward",
+            "method_output_name": "last_hidden_state"
+        },
+        "audio": {
+            "method": "forward",
+            "method_output_name": "last_hidden_state"
+        },
+        "video": {
+            "method": "forward",
+            "method_output_name": "last_hidden_state"
+        },
+        "message": {
+            "method": "forward",
+            "method_output_name": "last_hidden_state",
+            "format": "structured"
+        }
+    },
+    "module_output_name": "token_embeddings"
+}

smoke.json

@@ -0,0 +1,54 @@
+{
+  "pytorch_neural_loading_disabled": true,
+  "cases": [
+    {
+      "modalities": [
+        "text",
+        "prompt_name"
+      ],
+      "shape": [
+        768
+      ],
+      "unit_norm": 1.0
+    },
+    {
+      "modalities": [
+        "images"
+      ],
+      "shape": [
+        768
+      ],
+      "unit_norm": 1.0
+    },
+    {
+      "modalities": [
+        "audio"
+      ],
+      "shape": [
+        768
+      ],
+      "unit_norm": 1.0
+    },
+    {
+      "modalities": [
+        "videos"
+      ],
+      "shape": [
+        768
+      ],
+      "unit_norm": 1.0
+    },
+    {
+      "modalities": [
+        "text",
+        "images",
+        "audio"
+      ],
+      "shape": [
+        768
+      ],
+      "unit_norm": 1.0
+    }
+  ],
+  "invalid_inputs_rejected": 4
+}

smoke_gemma2.py

@@ -0,0 +1,78 @@
+"""Run with no source checkpoint and disable PyTorch neural weight loading."""
+
+import asyncio
+import json
+from pathlib import Path
+
+import numpy as np
+from PIL import Image
+from transformers import PreTrainedModel
+
+from coreai_embeddinggemma2 import EmbeddingGemma2CoreAI
+
+
+def forbidden(*args, **kwargs):
+    raise AssertionError(
+        "PyTorch neural weight loading is forbidden in this smoke test"
+    )
+
+
+PreTrainedModel.from_pretrained = classmethod(forbidden)
+
+
+async def main():
+    root = Path("bundles/embeddinggemma-2-CoreAI")
+    runtime = await EmbeddingGemma2CoreAI.load(root)
+    audio = (0.2 * np.sin(2 * np.pi * 440 * np.arange(8000) / 16000)).astype(np.float32)
+    image = Image.new("RGB", (64, 96), (30, 70, 200))
+    video = np.stack([np.asarray(image), np.asarray(image)])
+    inputs = [
+        {"text": "What causes auroras?", "prompt_name": "SearchQuery"},
+        {"images": image},
+        {"audio": audio},
+        {"videos": video},
+        {"text": "<|image|> Blue. <|audio|> Tone.", "images": image, "audio": audio},
+    ]
+    results = []
+    for data in inputs:
+        v = await runtime.encode(**data)
+        assert (
+            v.shape == (768,)
+            and np.isfinite(v).all()
+            and abs(np.linalg.norm(v) - 1) < 1e-5
+        )
+        results.append(
+            {
+                "modalities": list(data),
+                "shape": list(v.shape),
+                "unit_norm": float(np.linalg.norm(v)),
+            }
+        )
+    for data in [
+        {"text": "word " * 3000},
+        {"audio": np.zeros(179201, np.float32)},
+        {"audio": audio, "sampling_rate": 8000},
+        {"text": "hello", "truncate_dim": 769},
+    ]:
+        try:
+            await runtime.encode(**data)
+        except ValueError:
+            pass
+        else:
+            raise AssertionError("Invalid input accepted")
+    (root / "smoke.json").write_text(
+        json.dumps(
+            {
+                "pytorch_neural_loading_disabled": True,
+                "cases": results,
+                "invalid_inputs_rejected": 4,
+            },
+            indent=2,
+        )
+        + "\n"
+    )
+    print("Core AI runtime-only smoke passed", flush=True)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4
+size 32170510

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e594c8a90eb08d8bda498ff4747977dc827ae0c3c56b5c0d41a605a22d02ef03
+size 4689013

tokenizer_config.json

@@ -0,0 +1,57 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|video|>"
+  ],
+  "image_token": "<|image|>",
+  "is_local": true,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "audio_token": "<|audio|>",
+    "boa_token": "<|audio>",
+    "boi_token": "<|image>",
+    "eoa_token": "<audio|>",
+    "eoc_token": "<channel|>",
+    "eoi_token": "<image|>",
+    "eot_token": "<turn|>",
+    "escape_token": "<|\"|>",
+    "etc_token": "<tool_call|>",
+    "etd_token": "<tool|>",
+    "etr_token": "<tool_response|>",
+    "image_token": "<|image|>",
+    "soc_token": "<|channel>",
+    "sot_token": "<|turn>",
+    "stc_token": "<|tool_call>",
+    "std_token": "<|tool>",
+    "str_token": "<|tool_response>",
+    "think_token": "<|think|>",
+    "video_token": "<|video|>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "processor_class": "EmbeddingGemma2Processor",
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "video_token": "<|video|>"
+}

validate_gemma2.py

@@ -0,0 +1,148 @@
+import asyncio
+import json
+import shutil
+from pathlib import Path
+
+import numpy as np
+import torch
+from PIL import Image
+from transformers import AutoModel, AutoProcessor
+
+from coreai_embeddinggemma2 import EmbeddingGemma2CoreAI
+
+
+def cases():
+    wave = (0.2 * np.sin(2 * np.pi * 440 * np.arange(16000) / 16000)).astype(np.float32)
+    image = Image.new("RGB", (320, 192), (20, 80, 210))
+    gradient = Image.fromarray(
+        np.tile(np.arange(256, dtype=np.uint8)[None, :, None], (128, 1, 3))
+    )
+    frames = np.stack(
+        [
+            np.asarray(image.resize((128, 96))),
+            np.asarray(Image.new("RGB", (128, 96), (210, 50, 20))),
+        ]
+    )
+    return [
+        (
+            "query",
+            {"text": "What causes the northern lights?", "prompt_name": "SearchQuery"},
+        ),
+        (
+            "document",
+            {
+                "text": "The northern lights are caused by charged particles from the sun.",
+                "prompt_name": "Document",
+            },
+        ),
+        ("unprefixed", {"text": "An apple a day keeps the doctor away."}),
+        (
+            "code",
+            {"text": "def add(a, b): return a + b", "prompt_name": "CodeRetrieval"},
+        ),
+        ("spanish", {"text": "La biblioteca está abierta hasta las ocho."}),
+        ("japanese", {"text": "桜の花が春に咲きます。"}),
+        ("short", {"text": "cat"}),
+        ("long", {"text": "The oceans contain diverse ecosystems. " * 90}),
+        ("near-capacity", {"text": "The oceans contain diverse ecosystems. " * 160}),
+        ("image", {"images": image}),
+        ("gradient", {"images": gradient}),
+        ("image-text", {"text": "<|image|> A blue rectangle.", "images": image}),
+        ("two-images", {"images": [image, gradient]}),
+        ("audio", {"audio": wave}),
+        ("audio-text", {"text": "<|audio|> A musical tone.", "audio": wave}),
+        ("audio-short", {"audio": wave[:3200]}),
+        ("audio-long", {"audio": np.tile(wave, 5)}),
+        ("audio-limit", {"audio": np.pad(np.tile(wave, 11), (0, 3200))}),
+        ("video", {"videos": frames}),
+        ("video-text", {"text": "<|video|> Colors changing.", "videos": frames}),
+        (
+            "mixed",
+            {
+                "text": "<|image|> Blue. <|audio|> Tone. <|video|> Colors.",
+                "images": image,
+                "audio": wave,
+                "videos": frames,
+            },
+        ),
+    ]
+
+
+async def main():
+    torch.set_num_threads(4)
+    root = Path("bundles/embeddinggemma-2-CoreAI")
+    source = Path("upstream/embeddinggemma-2")
+    for f in source.iterdir():
+        if f.is_file() and f.suffix in [".json", ".jinja", ".model"]:
+            shutil.copy2(f, root / f.name)
+    m = AutoModel.from_pretrained(
+        source,
+        dtype=torch.float32,
+        attn_implementation={
+            "language_model": "sdpa",
+            "vision_tower": "sdpa",
+            "audio_tower": "eager",
+        },
+    ).eval()
+    p = AutoProcessor.from_pretrained(source)
+    print("Loading Core AI", flush=True)
+    runtime = await EmbeddingGemma2CoreAI.load(root)
+    print("Loaded Core AI", flush=True)
+    records = []
+    for name, args in cases():
+        data = dict(args)
+        prompt = data.pop("prompt_name", None)
+        if prompt:
+            data["text"] = runtime.prompts[prompt] + data["text"]
+        if "audio" in data:
+            data["sampling_rate"] = 16000
+        if "videos" in data:
+            data["videos_kwargs"] = {"do_sample_frames": False}
+        e = p(**data, return_tensors="pt")
+        with torch.no_grad():
+            h = m(**e).last_hidden_state
+            mask = e["attention_mask"][:, :, None].float()
+            reference = torch.nn.functional.normalize(
+                (h * mask).sum(1) / mask.sum(1), dim=-1
+            )[0].numpy()
+        actual = await runtime.encode_processed({k: v.numpy() for k, v in e.items()})
+        record = {
+            "name": name,
+            "tokens": e["input_ids"].shape[1],
+            "max_abs_error": float(np.max(np.abs(actual - reference))),
+            "cosine": float(actual @ reference),
+            "finite": bool(np.isfinite(actual).all()),
+            "unit_norm": float(np.linalg.norm(actual)),
+        }
+        print(record, flush=True)
+        records.append(record)
+        assert (
+            record["finite"]
+            and record["cosine"] > 0.99999
+            and record["max_abs_error"] < 0.00005
+        ), record
+    for dim in [128, 256, 512]:
+        actual = await runtime.encode("The sky is blue.", truncate_dim=dim)
+        assert actual.shape == (dim,) and abs(np.linalg.norm(actual) - 1) < 1e-5
+    try:
+        await runtime.encode("word " * 3000)
+    except ValueError:
+        pass
+    else:
+        raise AssertionError("Over-capacity input accepted")
+    (root / "validation.json").write_text(
+        json.dumps(
+            {
+                "thresholds": {"min_cosine": 0.99999, "max_abs_error": 0.00005},
+                "cases": records,
+                "truncation_dims": [128, 256, 512],
+                "oversized_input_rejected": True,
+            },
+            indent=2,
+        )
+        + "\n"
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

validation.json

@@ -0,0 +1,182 @@
+{
+  "thresholds": {
+    "min_cosine": 0.99999,
+    "max_abs_error": 5e-05
+  },
+  "cases": [
+    {
+      "name": "query",
+      "tokens": 15,
+      "max_abs_error": 1.0617077350616455e-07,
+      "cosine": 0.9999999403953552,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "document",
+      "tokens": 20,
+      "max_abs_error": 1.2968666851520538e-07,
+      "cosine": 1.0,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "unprefixed",
+      "tokens": 11,
+      "max_abs_error": 2.086162567138672e-07,
+      "cosine": 0.9999998807907104,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "code",
+      "tokens": 20,
+      "max_abs_error": 1.8719583749771118e-07,
+      "cosine": 1.0,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "spanish",
+      "tokens": 10,
+      "max_abs_error": 1.0803341865539551e-07,
+      "cosine": 1.0000001192092896,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "japanese",
+      "tokens": 10,
+      "max_abs_error": 1.043081283569336e-07,
+      "cosine": 1.0,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "short",
+      "tokens": 3,
+      "max_abs_error": 7.450580596923828e-08,
+      "cosine": 1.0000001192092896,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "long",
+      "tokens": 543,
+      "max_abs_error": 1.46450474858284e-07,
+      "cosine": 0.9999998807907104,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "near-capacity",
+      "tokens": 963,
+      "max_abs_error": 1.175212673842907e-07,
+      "cosine": 1.0,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "image",
+      "tokens": 256,
+      "max_abs_error": 6.705522537231445e-08,
+      "cosine": 1.0000001192092896,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "gradient",
+      "tokens": 257,
+      "max_abs_error": 8.940696716308594e-08,
+      "cosine": 0.9999999403953552,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "image-text",
+      "tokens": 260,
+      "max_abs_error": 1.1548399925231934e-07,
+      "cosine": 1.0000001192092896,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "two-images",
+      "tokens": 512,
+      "max_abs_error": 1.8440186977386475e-07,
+      "cosine": 1.0,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "audio",
+      "tokens": 29,
+      "max_abs_error": 3.3155083656311035e-07,
+      "cosine": 1.000000238418579,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "audio-text",
+      "tokens": 33,
+      "max_abs_error": 2.4493783712387085e-07,
+      "cosine": 1.0,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "audio-short",
+      "tokens": 9,
+      "max_abs_error": 1.7462298274040222e-07,
+      "cosine": 1.0000001192092896,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "audio-long",
+      "tokens": 129,
+      "max_abs_error": 7.130438461899757e-07,
+      "cosine": 1.0,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "audio-limit",
+      "tokens": 284,
+      "max_abs_error": 9.760260581970215e-07,
+      "cosine": 0.9999999403953552,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "video",
+      "tokens": 266,
+      "max_abs_error": 1.126900315284729e-07,
+      "cosine": 1.0000001192092896,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "video-text",
+      "tokens": 269,
+      "max_abs_error": 1.341104507446289e-07,
+      "cosine": 1.0,
+      "finite": true,
+      "unit_norm": 1.0
+    },
+    {
+      "name": "mixed",
+      "tokens": 555,
+      "max_abs_error": 2.0302832126617432e-07,
+      "cosine": 1.0000001192092896,
+      "finite": true,
+      "unit_norm": 1.0
+    }
+  ],
+  "truncation_dims": [
+    128,
+    256,
+    512
+  ],
+  "oversized_input_rejected": true
+}

vision.aimodel/main.hash

@@ -0,0 +1 @@
+ʿ��+#�\������-8mC�S��-�i5�x

vision.aimodel/main.mlirb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cabf97de2b23a35cecea87c802f4f9f12d0b38126d43af539e932ddf6935f978
+size 671516334

vision.aimodel/metadata.json

@@ -0,0 +1,5 @@
+{
+  "assetVersion" : "2.0",
+  "creationDate" : "20261011T130707Z",
+  "producer" : "coreai-core 1.0.0b3"
+}