Integrate with Sentence Transformers via MultiVectorEncoder

1_Dense/config.json

@@ -0,0 +1,8 @@
+{
+  "in_features": 2560,
+  "out_features": 128,
+  "bias": true,
+  "activation_function": "torch.nn.modules.linear.Identity",
+  "module_input_name": "token_embeddings",
+  "module_output_name": "token_embeddings"
+}

1_Dense/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dc6f35f67a99e1146d9464051d2cbc8bb5c4b7085ebcf95ba45cf285793a346
+size 1311392

2_Normalize/config.json

@@ -0,0 +1,4 @@
+{
+  "module_input_name": "token_embeddings",
+  "module_output_name": "token_embeddings"
+}

3_MultiVectorMask/config.json

@@ -0,0 +1,5 @@
+{
+  "skiplist_words": [],
+  "skiplist_tasks": [],
+  "keep_only_token_ids": null
+}

README.md

@@ -18,6 +18,7 @@ tags:
   - qwen3-vl
   - mteb
   - vidore
+  - sentence-transformers
 base_model: Qwen/Qwen3-VL-4B
 pipeline_tag: image-text-to-text
 datasets:
@@ -199,6 +200,45 @@ Traditional OCR-based retrieval **loses layout, tables, and visual context**. Ou
 
 ## Installation & Usage
 
+### Sentence Transformers
+
+This model can be used with [Sentence Transformers](https://www.sbert.net/) as a multi-vector (ColBERT-style late interaction) retriever via the `MultiVectorEncoder`:
+
+```bash
+pip install "sentence-transformers[image]>=6.0.0"
+```
+
+```python
+from sentence_transformers import MultiVectorEncoder
+
+model = MultiVectorEncoder(
+    "VAGOsolutions/SauerkrautLM-ColQwen3-4b-v0.1",
+    model_kwargs={"dtype": "bfloat16"},
+)
+
+queries = [
+    "What is the variable represented on the y-axis of the graph?",
+    "Total outlay is maximum in which year?",
+]
+images = [
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc1.jpg",
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc2.jpg",
+]
+
+query_embeddings = model.encode_query(queries)
+image_embeddings = model.encode_document(images)
+print(query_embeddings[0].shape, image_embeddings[0].shape)
+# torch.Size([25, 128]) torch.Size([1251, 128])
+
+# Diagonal should have higher scores
+scores = model.similarity(query_embeddings, image_embeddings)
+print(scores)
+# tensor([[16.3877,  8.1367],
+#         [ 5.8350, 15.3848]], device='cuda:0')
+```
+
+### SauerkrautLM ColPali
+
 > ⚠️ **Important**: Install our package first before loading the model:
 
 ```bash

additional_chat_templates/sentence_transformers.jinja

@@ -0,0 +1,17 @@
+{%- for message in messages -%}
+{%- set ns = namespace(text='') -%}
+{%- if message['content'] is string -%}
+{%- set ns.text = message['content'] -%}
+{%- else -%}
+{%- for item in message['content'] -%}
+{%- if 'text' in item -%}
+{%- set ns.text = ns.text + item.text -%}
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
+{%- if task is defined and task == 'query' -%}
+{{- 'Query: ' + ns.text + '<|endoftext|>' * 10 -}}
+{%- else -%}
+{{- '<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe the image.<|im_end|><|endoftext|>' -}}
+{%- endif -%}
+{%- endfor -%}

config_sentence_transformers.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "MultiVectorEncoder",
+  "similarity_fn_name": "maxsim",
+  "prompts": {},
+  "default_prompt_name": null,
+  "__version__": {
+    "sentence_transformers": "6.0.0"
+  }
+}

modules.json

@@ -0,0 +1,26 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.base.modules.transformer.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Dense",
+    "type": "sentence_transformers.base.modules.dense.Dense"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.base.modules.normalize.Normalize"
+  },
+  {
+    "idx": 3,
+    "name": "3",
+    "path": "3_MultiVectorMask",
+    "type": "sentence_transformers.multi_vector_encoder.modules.multi_vector_mask.MultiVectorMask"
+  }
+]

processor_config.json

@@ -0,0 +1,3 @@
+{
+  "processor_class": "Qwen3VLProcessor"
+}

sentence_bert_config.json

@@ -0,0 +1,27 @@
+{
+  "transformer_task": "feature-extraction",
+  "modality_config": {
+    "text": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state"
+    },
+    "image": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state"
+    },
+    "message": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state",
+      "format": "structured"
+    }
+  },
+  "module_output_name": "token_embeddings",
+  "processing_kwargs": {
+    "chat_template": {
+      "chat_template": "sentence_transformers"
+    },
+    "text": {
+      "return_mm_token_type_ids": true
+    }
+  }
+}

tokenizer_config.json

@@ -237,5 +237,6 @@
   "processor_class": "ColQwen3Processor",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  "unk_token": null,
+  "padding_side": "left"
 }