Integrate with Sentence Transformers via MultiVectorEncoder

1_Dense/config.json

@@ -0,0 +1,8 @@
+{
+  "in_features": 1024,
+  "out_features": 128,
+  "bias": true,
+  "activation_function": "torch.nn.modules.linear.Identity",
+  "module_input_name": "token_embeddings",
+  "module_output_name": "token_embeddings"
+}

1_Dense/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e8a23b489bfbea8795e6d76e1c2e912155cdd6e3b556a6da61bbe17200c18e3
+size 262560

2_Normalize/config.json

@@ -0,0 +1,4 @@
+{
+  "module_input_name": "token_embeddings",
+  "module_output_name": "token_embeddings"
+}

3_MultiVectorMask/config.json

@@ -0,0 +1,5 @@
+{
+  "skiplist_words": [],
+  "skiplist_tasks": [],
+  "keep_only_token_ids": null
+}

README.md

@@ -24,6 +24,7 @@ tags:
 - hierarchical-merge
 - mteb
 - vidore
+- sentence-transformers
 base_model: LiquidAI/LFM2-VL-450M
 pipeline_tag: image-text-to-text
 datasets:
@@ -241,6 +242,42 @@ Specialist  Model
 
 ## Installation & Usage
 
+### Sentence Transformers
+
+This model can be used with [Sentence Transformers](https://www.sbert.net/) as a multi-vector (ColBERT-style late interaction) retriever via the `MultiVectorEncoder`:
+
+```bash
+pip install "sentence-transformers[image]>=6.0.0"
+```
+
+```python
+from sentence_transformers import MultiVectorEncoder
+
+model = MultiVectorEncoder("VAGOsolutions/SauerkrautLM-ColLFM2-450M-v0.1")
+
+queries = [
+    "What is the variable represented on the y-axis of the graph?",
+    "Total outlay is maximum in which year?",
+]
+images = [
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc1.jpg",
+    "https://huggingface.co/datasets/sentence-transformers/example-documents/resolve/main/doc2.jpg",
+]
+
+query_embeddings = model.encode_query(queries)
+image_embeddings = model.encode_document(images)
+print(query_embeddings[0].shape, image_embeddings[0].shape)
+# torch.Size([14, 128]) torch.Size([1792, 128])
+
+# Diagonal should have higher scores
+scores = model.similarity(query_embeddings, image_embeddings)
+print(scores)
+# tensor([[13.5820, 13.4766],
+#         [ 9.2461,  9.5703]], device='cuda:0')
+```
+
+### SauerkrautLM ColPali
+
 > ⚠️ **Important**: Install our package first before loading the model:
 
 ```bash

additional_chat_templates/sentence_transformers.jinja

@@ -0,0 +1,17 @@
+{%- for message in messages -%}
+{%- set ns = namespace(text='') -%}
+{%- if message['content'] is string -%}
+{%- set ns.text = message['content'] -%}
+{%- else -%}
+{%- for item in message['content'] -%}
+{%- if 'text' in item -%}
+{%- set ns.text = ns.text + item.text -%}
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
+{%- if task is defined and task == 'query' -%}
+{{- bos_token + ns.text -}}
+{%- else -%}
+{{- '<|im_start|>user\n<image>Describe the image.<|im_end|>' -}}
+{%- endif -%}
+{%- endfor -%}

config_sentence_transformers.json

@@ -0,0 +1,9 @@
+{
+  "model_type": "MultiVectorEncoder",
+  "similarity_fn_name": "maxsim",
+  "prompts": {},
+  "default_prompt_name": null,
+  "__version__": {
+    "sentence_transformers": "6.0.0"
+  }
+}

modules.json

@@ -0,0 +1,26 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.base.modules.transformer.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Dense",
+    "type": "sentence_transformers.base.modules.dense.Dense"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.base.modules.normalize.Normalize"
+  },
+  {
+    "idx": 3,
+    "name": "3",
+    "path": "3_MultiVectorMask",
+    "type": "sentence_transformers.multi_vector_encoder.modules.multi_vector_mask.MultiVectorMask"
+  }
+]

sentence_bert_config.json

@@ -0,0 +1,116 @@
+{
+  "transformer_task": "feature-extraction",
+  "modality_config": {
+    "text": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state"
+    },
+    "image": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state"
+    },
+    "message": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state",
+      "format": "structured"
+    }
+  },
+  "module_output_name": "token_embeddings",
+  "processing_kwargs": {
+    "chat_template": {
+      "chat_template": "sentence_transformers"
+    }
+  },
+  "config_kwargs": {
+    "do_image_splitting": true,
+    "downsample_factor": 2,
+    "encoder_patch_size": 16,
+    "image_token_id": 396,
+    "image_token_index": 396,
+    "max_image_tokens": 256,
+    "max_num_patches": 1024,
+    "max_pixels_tolerance": 2.0,
+    "max_tiles": 10,
+    "min_image_tokens": 64,
+    "min_tiles": 2,
+    "model_type": "lfm2_vl",
+    "projector_bias": true,
+    "projector_hidden_act": "gelu",
+    "projector_hidden_size": 2560,
+    "text_config": {
+      "architectures": [
+        "Lfm2ForCausalLM"
+      ],
+      "block_auto_adjust_ff_dim": true,
+      "block_dim": 1024,
+      "block_ff_dim": 6656,
+      "block_ffn_dim_multiplier": 1.0,
+      "block_mlp_init_scale": 1.0,
+      "block_multiple_of": 256,
+      "block_norm_eps": 1e-05,
+      "block_out_init_scale": 1.0,
+      "block_use_swiglu": true,
+      "block_use_xavier_init": true,
+      "conv_L_cache": 3,
+      "conv_bias": false,
+      "conv_dim": 1024,
+      "conv_dim_out": 1024,
+      "conv_use_xavier_init": true,
+      "eos_token_id": 7,
+      "hidden_size": 1024,
+      "initializer_range": 0.02,
+      "intermediate_size": 6656,
+      "layer_types": [
+        "conv",
+        "conv",
+        "full_attention",
+        "conv",
+        "conv",
+        "full_attention",
+        "conv",
+        "conv",
+        "full_attention",
+        "conv",
+        "full_attention",
+        "conv",
+        "full_attention",
+        "conv",
+        "full_attention",
+        "conv"
+      ],
+      "max_position_embeddings": 128000,
+      "model_type": "lfm2",
+      "norm_eps": 1e-05,
+      "num_attention_heads": 16,
+      "num_heads": 16,
+      "num_hidden_layers": 16,
+      "num_key_value_heads": 8,
+      "rope_theta": 1000000.0,
+      "use_cache": true,
+      "use_pos_enc": true,
+      "vocab_size": 65536
+    },
+    "tile_size": 512,
+    "use_image_special_tokens": true,
+    "use_thumbnail": false,
+    "vision_config": {
+      "attention_dropout": 0.0,
+      "hidden_act": "gelu_pytorch_tanh",
+      "hidden_size": 768,
+      "intermediate_size": 3072,
+      "layer_norm_eps": 1e-06,
+      "model_type": "siglip2_vision_model",
+      "num_attention_heads": 12,
+      "num_channels": 3,
+      "num_hidden_layers": 12,
+      "num_patches": 256,
+      "patch_size": 16,
+      "vision_use_head": false
+    }
+  },
+  "model_kwargs": {
+    "key_mapping": {
+      "^model\\.": ""
+    }
+  }
+}

tokenizer_config.json

@@ -4084,5 +4084,6 @@
   "spaces_between_special_tokens": false,
   "tokenizer_class": "PreTrainedTokenizerFast",
   "use_default_system_prompt": false,
-  "use_fast": true
+  "use_fast": true,
+  "padding_side": "left"
 }