Hugging Face's logo

Jonandrop
/
Ornith-1.0-35B-MTPLX-Vision

Image-Text-to-Text
MLX
Safetensors
qwen3_5_moe
apple-silicon
speculative-decoding
qwen
qwen3
qwen3-next
mtp
mtplx
local-ai
vision-language
multimodal
conversational
Model card Files
xet
 Community
Jonandrop commited on
Commit
c30be3c
·
verified ·
1 Parent(s): b216279

Upload config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +20 -674
config.json CHANGED
@@ -4,682 +4,11 @@
4
  ],
5
  "bos_token_id": null,
6
  "dtype": "bfloat16",
7
- "eos_token_id": [
8
- 248046,
9
- 248044
10
- ],
11
  "hidden_size": 2048,
12
  "image_token_id": 248056,
13
- "mlx_lm_extra_tensors": {
14
- "mtp_file": "mtp.safetensors"
15
- },
16
  "model_type": "qwen3_5_moe",
17
- "mtplx_mtp_contract": {
18
- "base_hidden_variant": "post_norm",
19
- "concat_order": "embedding_hidden",
20
- "hidden_variant": "post_norm",
21
- "mtp_position_mode": "local",
22
- "mtp_quant_bits": 4,
23
- "mtp_quant_group_size": 64,
24
- "mtp_quant_mode": "affine"
25
- },
26
- "mtplx_mtp_norm_encoding": "delta",
27
- "mtplx_mtp_quantization": {
28
- "bits": 4,
29
- "group_size": 64,
30
- "mode": "affine"
31
- },
32
  "pad_token_id": 248044,
33
- "quantization": {
34
- "bits": 4,
35
- "group_size": 64,
36
- "language_model.model.layers.0.mlp.gate": {
37
- "bits": 8,
38
- "group_size": 64
39
- },
40
- "language_model.model.layers.0.mlp.shared_expert_gate": {
41
- "bits": 8,
42
- "group_size": 64
43
- },
44
- "language_model.model.layers.1.mlp.gate": {
45
- "bits": 8,
46
- "group_size": 64
47
- },
48
- "language_model.model.layers.1.mlp.shared_expert_gate": {
49
- "bits": 8,
50
- "group_size": 64
51
- },
52
- "language_model.model.layers.10.mlp.gate": {
53
- "bits": 8,
54
- "group_size": 64
55
- },
56
- "language_model.model.layers.10.mlp.shared_expert_gate": {
57
- "bits": 8,
58
- "group_size": 64
59
- },
60
- "language_model.model.layers.11.mlp.gate": {
61
- "bits": 8,
62
- "group_size": 64
63
- },
64
- "language_model.model.layers.11.mlp.shared_expert_gate": {
65
- "bits": 8,
66
- "group_size": 64
67
- },
68
- "language_model.model.layers.12.mlp.gate": {
69
- "bits": 8,
70
- "group_size": 64
71
- },
72
- "language_model.model.layers.12.mlp.shared_expert_gate": {
73
- "bits": 8,
74
- "group_size": 64
75
- },
76
- "language_model.model.layers.13.mlp.gate": {
77
- "bits": 8,
78
- "group_size": 64
79
- },
80
- "language_model.model.layers.13.mlp.shared_expert_gate": {
81
- "bits": 8,
82
- "group_size": 64
83
- },
84
- "language_model.model.layers.14.mlp.gate": {
85
- "bits": 8,
86
- "group_size": 64
87
- },
88
- "language_model.model.layers.14.mlp.shared_expert_gate": {
89
- "bits": 8,
90
- "group_size": 64
91
- },
92
- "language_model.model.layers.15.mlp.gate": {
93
- "bits": 8,
94
- "group_size": 64
95
- },
96
- "language_model.model.layers.15.mlp.shared_expert_gate": {
97
- "bits": 8,
98
- "group_size": 64
99
- },
100
- "language_model.model.layers.16.mlp.gate": {
101
- "bits": 8,
102
- "group_size": 64
103
- },
104
- "language_model.model.layers.16.mlp.shared_expert_gate": {
105
- "bits": 8,
106
- "group_size": 64
107
- },
108
- "language_model.model.layers.17.mlp.gate": {
109
- "bits": 8,
110
- "group_size": 64
111
- },
112
- "language_model.model.layers.17.mlp.shared_expert_gate": {
113
- "bits": 8,
114
- "group_size": 64
115
- },
116
- "language_model.model.layers.18.mlp.gate": {
117
- "bits": 8,
118
- "group_size": 64
119
- },
120
- "language_model.model.layers.18.mlp.shared_expert_gate": {
121
- "bits": 8,
122
- "group_size": 64
123
- },
124
- "language_model.model.layers.19.mlp.gate": {
125
- "bits": 8,
126
- "group_size": 64
127
- },
128
- "language_model.model.layers.19.mlp.shared_expert_gate": {
129
- "bits": 8,
130
- "group_size": 64
131
- },
132
- "language_model.model.layers.2.mlp.gate": {
133
- "bits": 8,
134
- "group_size": 64
135
- },
136
- "language_model.model.layers.2.mlp.shared_expert_gate": {
137
- "bits": 8,
138
- "group_size": 64
139
- },
140
- "language_model.model.layers.20.mlp.gate": {
141
- "bits": 8,
142
- "group_size": 64
143
- },
144
- "language_model.model.layers.20.mlp.shared_expert_gate": {
145
- "bits": 8,
146
- "group_size": 64
147
- },
148
- "language_model.model.layers.21.mlp.gate": {
149
- "bits": 8,
150
- "group_size": 64
151
- },
152
- "language_model.model.layers.21.mlp.shared_expert_gate": {
153
- "bits": 8,
154
- "group_size": 64
155
- },
156
- "language_model.model.layers.22.mlp.gate": {
157
- "bits": 8,
158
- "group_size": 64
159
- },
160
- "language_model.model.layers.22.mlp.shared_expert_gate": {
161
- "bits": 8,
162
- "group_size": 64
163
- },
164
- "language_model.model.layers.23.mlp.gate": {
165
- "bits": 8,
166
- "group_size": 64
167
- },
168
- "language_model.model.layers.23.mlp.shared_expert_gate": {
169
- "bits": 8,
170
- "group_size": 64
171
- },
172
- "language_model.model.layers.24.mlp.gate": {
173
- "bits": 8,
174
- "group_size": 64
175
- },
176
- "language_model.model.layers.24.mlp.shared_expert_gate": {
177
- "bits": 8,
178
- "group_size": 64
179
- },
180
- "language_model.model.layers.25.mlp.gate": {
181
- "bits": 8,
182
- "group_size": 64
183
- },
184
- "language_model.model.layers.25.mlp.shared_expert_gate": {
185
- "bits": 8,
186
- "group_size": 64
187
- },
188
- "language_model.model.layers.26.mlp.gate": {
189
- "bits": 8,
190
- "group_size": 64
191
- },
192
- "language_model.model.layers.26.mlp.shared_expert_gate": {
193
- "bits": 8,
194
- "group_size": 64
195
- },
196
- "language_model.model.layers.27.mlp.gate": {
197
- "bits": 8,
198
- "group_size": 64
199
- },
200
- "language_model.model.layers.27.mlp.shared_expert_gate": {
201
- "bits": 8,
202
- "group_size": 64
203
- },
204
- "language_model.model.layers.28.mlp.gate": {
205
- "bits": 8,
206
- "group_size": 64
207
- },
208
- "language_model.model.layers.28.mlp.shared_expert_gate": {
209
- "bits": 8,
210
- "group_size": 64
211
- },
212
- "language_model.model.layers.29.mlp.gate": {
213
- "bits": 8,
214
- "group_size": 64
215
- },
216
- "language_model.model.layers.29.mlp.shared_expert_gate": {
217
- "bits": 8,
218
- "group_size": 64
219
- },
220
- "language_model.model.layers.3.mlp.gate": {
221
- "bits": 8,
222
- "group_size": 64
223
- },
224
- "language_model.model.layers.3.mlp.shared_expert_gate": {
225
- "bits": 8,
226
- "group_size": 64
227
- },
228
- "language_model.model.layers.30.mlp.gate": {
229
- "bits": 8,
230
- "group_size": 64
231
- },
232
- "language_model.model.layers.30.mlp.shared_expert_gate": {
233
- "bits": 8,
234
- "group_size": 64
235
- },
236
- "language_model.model.layers.31.mlp.gate": {
237
- "bits": 8,
238
- "group_size": 64
239
- },
240
- "language_model.model.layers.31.mlp.shared_expert_gate": {
241
- "bits": 8,
242
- "group_size": 64
243
- },
244
- "language_model.model.layers.32.mlp.gate": {
245
- "bits": 8,
246
- "group_size": 64
247
- },
248
- "language_model.model.layers.32.mlp.shared_expert_gate": {
249
- "bits": 8,
250
- "group_size": 64
251
- },
252
- "language_model.model.layers.33.mlp.gate": {
253
- "bits": 8,
254
- "group_size": 64
255
- },
256
- "language_model.model.layers.33.mlp.shared_expert_gate": {
257
- "bits": 8,
258
- "group_size": 64
259
- },
260
- "language_model.model.layers.34.mlp.gate": {
261
- "bits": 8,
262
- "group_size": 64
263
- },
264
- "language_model.model.layers.34.mlp.shared_expert_gate": {
265
- "bits": 8,
266
- "group_size": 64
267
- },
268
- "language_model.model.layers.35.mlp.gate": {
269
- "bits": 8,
270
- "group_size": 64
271
- },
272
- "language_model.model.layers.35.mlp.shared_expert_gate": {
273
- "bits": 8,
274
- "group_size": 64
275
- },
276
- "language_model.model.layers.36.mlp.gate": {
277
- "bits": 8,
278
- "group_size": 64
279
- },
280
- "language_model.model.layers.36.mlp.shared_expert_gate": {
281
- "bits": 8,
282
- "group_size": 64
283
- },
284
- "language_model.model.layers.37.mlp.gate": {
285
- "bits": 8,
286
- "group_size": 64
287
- },
288
- "language_model.model.layers.37.mlp.shared_expert_gate": {
289
- "bits": 8,
290
- "group_size": 64
291
- },
292
- "language_model.model.layers.38.mlp.gate": {
293
- "bits": 8,
294
- "group_size": 64
295
- },
296
- "language_model.model.layers.38.mlp.shared_expert_gate": {
297
- "bits": 8,
298
- "group_size": 64
299
- },
300
- "language_model.model.layers.39.mlp.gate": {
301
- "bits": 8,
302
- "group_size": 64
303
- },
304
- "language_model.model.layers.39.mlp.shared_expert_gate": {
305
- "bits": 8,
306
- "group_size": 64
307
- },
308
- "language_model.model.layers.4.mlp.gate": {
309
- "bits": 8,
310
- "group_size": 64
311
- },
312
- "language_model.model.layers.4.mlp.shared_expert_gate": {
313
- "bits": 8,
314
- "group_size": 64
315
- },
316
- "language_model.model.layers.5.mlp.gate": {
317
- "bits": 8,
318
- "group_size": 64
319
- },
320
- "language_model.model.layers.5.mlp.shared_expert_gate": {
321
- "bits": 8,
322
- "group_size": 64
323
- },
324
- "language_model.model.layers.6.mlp.gate": {
325
- "bits": 8,
326
- "group_size": 64
327
- },
328
- "language_model.model.layers.6.mlp.shared_expert_gate": {
329
- "bits": 8,
330
- "group_size": 64
331
- },
332
- "language_model.model.layers.7.mlp.gate": {
333
- "bits": 8,
334
- "group_size": 64
335
- },
336
- "language_model.model.layers.7.mlp.shared_expert_gate": {
337
- "bits": 8,
338
- "group_size": 64
339
- },
340
- "language_model.model.layers.8.mlp.gate": {
341
- "bits": 8,
342
- "group_size": 64
343
- },
344
- "language_model.model.layers.8.mlp.shared_expert_gate": {
345
- "bits": 8,
346
- "group_size": 64
347
- },
348
- "language_model.model.layers.9.mlp.gate": {
349
- "bits": 8,
350
- "group_size": 64
351
- },
352
- "language_model.model.layers.9.mlp.shared_expert_gate": {
353
- "bits": 8,
354
- "group_size": 64
355
- },
356
- "mode": "affine"
357
- },
358
- "quantization_config": {
359
- "bits": 4,
360
- "group_size": 64,
361
- "language_model.model.layers.0.mlp.gate": {
362
- "bits": 8,
363
- "group_size": 64
364
- },
365
- "language_model.model.layers.0.mlp.shared_expert_gate": {
366
- "bits": 8,
367
- "group_size": 64
368
- },
369
- "language_model.model.layers.1.mlp.gate": {
370
- "bits": 8,
371
- "group_size": 64
372
- },
373
- "language_model.model.layers.1.mlp.shared_expert_gate": {
374
- "bits": 8,
375
- "group_size": 64
376
- },
377
- "language_model.model.layers.10.mlp.gate": {
378
- "bits": 8,
379
- "group_size": 64
380
- },
381
- "language_model.model.layers.10.mlp.shared_expert_gate": {
382
- "bits": 8,
383
- "group_size": 64
384
- },
385
- "language_model.model.layers.11.mlp.gate": {
386
- "bits": 8,
387
- "group_size": 64
388
- },
389
- "language_model.model.layers.11.mlp.shared_expert_gate": {
390
- "bits": 8,
391
- "group_size": 64
392
- },
393
- "language_model.model.layers.12.mlp.gate": {
394
- "bits": 8,
395
- "group_size": 64
396
- },
397
- "language_model.model.layers.12.mlp.shared_expert_gate": {
398
- "bits": 8,
399
- "group_size": 64
400
- },
401
- "language_model.model.layers.13.mlp.gate": {
402
- "bits": 8,
403
- "group_size": 64
404
- },
405
- "language_model.model.layers.13.mlp.shared_expert_gate": {
406
- "bits": 8,
407
- "group_size": 64
408
- },
409
- "language_model.model.layers.14.mlp.gate": {
410
- "bits": 8,
411
- "group_size": 64
412
- },
413
- "language_model.model.layers.14.mlp.shared_expert_gate": {
414
- "bits": 8,
415
- "group_size": 64
416
- },
417
- "language_model.model.layers.15.mlp.gate": {
418
- "bits": 8,
419
- "group_size": 64
420
- },
421
- "language_model.model.layers.15.mlp.shared_expert_gate": {
422
- "bits": 8,
423
- "group_size": 64
424
- },
425
- "language_model.model.layers.16.mlp.gate": {
426
- "bits": 8,
427
- "group_size": 64
428
- },
429
- "language_model.model.layers.16.mlp.shared_expert_gate": {
430
- "bits": 8,
431
- "group_size": 64
432
- },
433
- "language_model.model.layers.17.mlp.gate": {
434
- "bits": 8,
435
- "group_size": 64
436
- },
437
- "language_model.model.layers.17.mlp.shared_expert_gate": {
438
- "bits": 8,
439
- "group_size": 64
440
- },
441
- "language_model.model.layers.18.mlp.gate": {
442
- "bits": 8,
443
- "group_size": 64
444
- },
445
- "language_model.model.layers.18.mlp.shared_expert_gate": {
446
- "bits": 8,
447
- "group_size": 64
448
- },
449
- "language_model.model.layers.19.mlp.gate": {
450
- "bits": 8,
451
- "group_size": 64
452
- },
453
- "language_model.model.layers.19.mlp.shared_expert_gate": {
454
- "bits": 8,
455
- "group_size": 64
456
- },
457
- "language_model.model.layers.2.mlp.gate": {
458
- "bits": 8,
459
- "group_size": 64
460
- },
461
- "language_model.model.layers.2.mlp.shared_expert_gate": {
462
- "bits": 8,
463
- "group_size": 64
464
- },
465
- "language_model.model.layers.20.mlp.gate": {
466
- "bits": 8,
467
- "group_size": 64
468
- },
469
- "language_model.model.layers.20.mlp.shared_expert_gate": {
470
- "bits": 8,
471
- "group_size": 64
472
- },
473
- "language_model.model.layers.21.mlp.gate": {
474
- "bits": 8,
475
- "group_size": 64
476
- },
477
- "language_model.model.layers.21.mlp.shared_expert_gate": {
478
- "bits": 8,
479
- "group_size": 64
480
- },
481
- "language_model.model.layers.22.mlp.gate": {
482
- "bits": 8,
483
- "group_size": 64
484
- },
485
- "language_model.model.layers.22.mlp.shared_expert_gate": {
486
- "bits": 8,
487
- "group_size": 64
488
- },
489
- "language_model.model.layers.23.mlp.gate": {
490
- "bits": 8,
491
- "group_size": 64
492
- },
493
- "language_model.model.layers.23.mlp.shared_expert_gate": {
494
- "bits": 8,
495
- "group_size": 64
496
- },
497
- "language_model.model.layers.24.mlp.gate": {
498
- "bits": 8,
499
- "group_size": 64
500
- },
501
- "language_model.model.layers.24.mlp.shared_expert_gate": {
502
- "bits": 8,
503
- "group_size": 64
504
- },
505
- "language_model.model.layers.25.mlp.gate": {
506
- "bits": 8,
507
- "group_size": 64
508
- },
509
- "language_model.model.layers.25.mlp.shared_expert_gate": {
510
- "bits": 8,
511
- "group_size": 64
512
- },
513
- "language_model.model.layers.26.mlp.gate": {
514
- "bits": 8,
515
- "group_size": 64
516
- },
517
- "language_model.model.layers.26.mlp.shared_expert_gate": {
518
- "bits": 8,
519
- "group_size": 64
520
- },
521
- "language_model.model.layers.27.mlp.gate": {
522
- "bits": 8,
523
- "group_size": 64
524
- },
525
- "language_model.model.layers.27.mlp.shared_expert_gate": {
526
- "bits": 8,
527
- "group_size": 64
528
- },
529
- "language_model.model.layers.28.mlp.gate": {
530
- "bits": 8,
531
- "group_size": 64
532
- },
533
- "language_model.model.layers.28.mlp.shared_expert_gate": {
534
- "bits": 8,
535
- "group_size": 64
536
- },
537
- "language_model.model.layers.29.mlp.gate": {
538
- "bits": 8,
539
- "group_size": 64
540
- },
541
- "language_model.model.layers.29.mlp.shared_expert_gate": {
542
- "bits": 8,
543
- "group_size": 64
544
- },
545
- "language_model.model.layers.3.mlp.gate": {
546
- "bits": 8,
547
- "group_size": 64
548
- },
549
- "language_model.model.layers.3.mlp.shared_expert_gate": {
550
- "bits": 8,
551
- "group_size": 64
552
- },
553
- "language_model.model.layers.30.mlp.gate": {
554
- "bits": 8,
555
- "group_size": 64
556
- },
557
- "language_model.model.layers.30.mlp.shared_expert_gate": {
558
- "bits": 8,
559
- "group_size": 64
560
- },
561
- "language_model.model.layers.31.mlp.gate": {
562
- "bits": 8,
563
- "group_size": 64
564
- },
565
- "language_model.model.layers.31.mlp.shared_expert_gate": {
566
- "bits": 8,
567
- "group_size": 64
568
- },
569
- "language_model.model.layers.32.mlp.gate": {
570
- "bits": 8,
571
- "group_size": 64
572
- },
573
- "language_model.model.layers.32.mlp.shared_expert_gate": {
574
- "bits": 8,
575
- "group_size": 64
576
- },
577
- "language_model.model.layers.33.mlp.gate": {
578
- "bits": 8,
579
- "group_size": 64
580
- },
581
- "language_model.model.layers.33.mlp.shared_expert_gate": {
582
- "bits": 8,
583
- "group_size": 64
584
- },
585
- "language_model.model.layers.34.mlp.gate": {
586
- "bits": 8,
587
- "group_size": 64
588
- },
589
- "language_model.model.layers.34.mlp.shared_expert_gate": {
590
- "bits": 8,
591
- "group_size": 64
592
- },
593
- "language_model.model.layers.35.mlp.gate": {
594
- "bits": 8,
595
- "group_size": 64
596
- },
597
- "language_model.model.layers.35.mlp.shared_expert_gate": {
598
- "bits": 8,
599
- "group_size": 64
600
- },
601
- "language_model.model.layers.36.mlp.gate": {
602
- "bits": 8,
603
- "group_size": 64
604
- },
605
- "language_model.model.layers.36.mlp.shared_expert_gate": {
606
- "bits": 8,
607
- "group_size": 64
608
- },
609
- "language_model.model.layers.37.mlp.gate": {
610
- "bits": 8,
611
- "group_size": 64
612
- },
613
- "language_model.model.layers.37.mlp.shared_expert_gate": {
614
- "bits": 8,
615
- "group_size": 64
616
- },
617
- "language_model.model.layers.38.mlp.gate": {
618
- "bits": 8,
619
- "group_size": 64
620
- },
621
- "language_model.model.layers.38.mlp.shared_expert_gate": {
622
- "bits": 8,
623
- "group_size": 64
624
- },
625
- "language_model.model.layers.39.mlp.gate": {
626
- "bits": 8,
627
- "group_size": 64
628
- },
629
- "language_model.model.layers.39.mlp.shared_expert_gate": {
630
- "bits": 8,
631
- "group_size": 64
632
- },
633
- "language_model.model.layers.4.mlp.gate": {
634
- "bits": 8,
635
- "group_size": 64
636
- },
637
- "language_model.model.layers.4.mlp.shared_expert_gate": {
638
- "bits": 8,
639
- "group_size": 64
640
- },
641
- "language_model.model.layers.5.mlp.gate": {
642
- "bits": 8,
643
- "group_size": 64
644
- },
645
- "language_model.model.layers.5.mlp.shared_expert_gate": {
646
- "bits": 8,
647
- "group_size": 64
648
- },
649
- "language_model.model.layers.6.mlp.gate": {
650
- "bits": 8,
651
- "group_size": 64
652
- },
653
- "language_model.model.layers.6.mlp.shared_expert_gate": {
654
- "bits": 8,
655
- "group_size": 64
656
- },
657
- "language_model.model.layers.7.mlp.gate": {
658
- "bits": 8,
659
- "group_size": 64
660
- },
661
- "language_model.model.layers.7.mlp.shared_expert_gate": {
662
- "bits": 8,
663
- "group_size": 64
664
- },
665
- "language_model.model.layers.8.mlp.gate": {
666
- "bits": 8,
667
- "group_size": 64
668
- },
669
- "language_model.model.layers.8.mlp.shared_expert_gate": {
670
- "bits": 8,
671
- "group_size": 64
672
- },
673
- "language_model.model.layers.9.mlp.gate": {
674
- "bits": 8,
675
- "group_size": 64
676
- },
677
- "language_model.model.layers.9.mlp.shared_expert_gate": {
678
- "bits": 8,
679
- "group_size": 64
680
- },
681
- "mode": "affine"
682
- },
683
  "text_config": {
684
  "attention_bias": false,
685
  "attention_dropout": 0.0,
@@ -763,7 +92,7 @@
763
  ],
764
  "partial_rotary_factor": 0.25,
765
  "rope_theta": 10000000,
766
- "type": "default"
767
  },
768
  "router_aux_loss_coef": 0.0,
769
  "shared_expert_intermediate_size": 512,
@@ -775,6 +104,23 @@
775
  "transformers_version": "5.8.1",
776
  "use_cache": false,
777
  "video_token_id": 248057,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
778
  "vision_end_token_id": 248054,
779
  "vision_start_token_id": 248053
780
- }
 
4
  ],
5
  "bos_token_id": null,
6
  "dtype": "bfloat16",
7
+ "eos_token_id": 248046,
 
 
 
8
  "hidden_size": 2048,
9
  "image_token_id": 248056,
 
 
 
10
  "model_type": "qwen3_5_moe",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  "pad_token_id": 248044,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  "text_config": {
13
  "attention_bias": false,
14
  "attention_dropout": 0.0,
 
92
  ],
93
  "partial_rotary_factor": 0.25,
94
  "rope_theta": 10000000,
95
+ "rope_type": "default"
96
  },
97
  "router_aux_loss_coef": 0.0,
98
  "shared_expert_intermediate_size": 512,
 
104
  "transformers_version": "5.8.1",
105
  "use_cache": false,
106
  "video_token_id": 248057,
107
+ "vision_config": {
108
+ "deepstack_visual_indexes": [],
109
+ "depth": 27,
110
+ "dtype": "bfloat16",
111
+ "hidden_act": "gelu_pytorch_tanh",
112
+ "hidden_size": 1152,
113
+ "in_channels": 3,
114
+ "initializer_range": 0.02,
115
+ "intermediate_size": 4304,
116
+ "model_type": "qwen3_5_moe_vision",
117
+ "num_heads": 16,
118
+ "num_position_embeddings": 2304,
119
+ "out_hidden_size": 2048,
120
+ "patch_size": 16,
121
+ "spatial_merge_size": 2,
122
+ "temporal_patch_size": 2
123
+ },
124
  "vision_end_token_id": 248054,
125
  "vision_start_token_id": 248053
126
+ }