Add Qwen3-VL (Dense) language model implementation (#553)

* Added Qwen3-VL dense language model

* Added Qwen3-VL dense language model test

Author: vincentamato

mlx_lm/models/qwen3.py

@@ -141,8 +141,12 @@ def __call__(
         self,
         inputs: mx.array,
         cache=None,
+        input_embeddings: Optional[mx.array] = None,
     ):
-        h = self.embed_tokens(inputs)
+        if input_embeddings is not None:
+            h = input_embeddings
+        else:
+            h = self.embed_tokens(inputs)
 
         if cache is None:
             cache = [None] * len(self.layers)
@@ -167,8 +171,9 @@ def __call__(
         self,
         inputs: mx.array,
         cache=None,
+        input_embeddings: Optional[mx.array] = None,
     ):
-        out = self.model(inputs, cache)
+        out = self.model(inputs, cache, input_embeddings)
         if self.args.tie_word_embeddings:
             out = self.model.embed_tokens.as_linear(out)
         else:

mlx_lm/models/qwen3_vl.py

@@ -0,0 +1,57 @@
+# Copyright © 2025 Apple Inc.
+
+from dataclasses import dataclass
+from typing import Optional
+
+import mlx.core as mx
+import mlx.nn as nn
+from mlx.utils import tree_flatten, tree_unflatten
+
+from . import qwen3
+from .base import BaseModelArgs
+
+
+@dataclass
+class ModelArgs(BaseModelArgs):
+    model_type: str
+    text_config: dict
+
+    @classmethod
+    def from_dict(cls, params):
+        if "text_config" not in params:
+            return cls(model_type=params["model_type"], text_config=params)
+        return super().from_dict(params)
+
+
+class Model(nn.Module):
+    def __init__(self, args: ModelArgs):
+        super().__init__()
+        self.args = args
+        self.model_type = args.model_type
+        self.language_model = qwen3.Model(qwen3.ModelArgs.from_dict(args.text_config))
+
+    def __call__(
+        self,
+        inputs: mx.array,
+        cache=None,
+        input_embeddings: Optional[mx.array] = None,
+    ):
+        return self.language_model(
+            inputs, cache=cache, input_embeddings=input_embeddings
+        )
+
+    def sanitize(self, weights):
+        weights = tree_unflatten(list(weights.items()))
+        weights.pop("vision_tower", None)
+        weights = dict(tree_flatten(weights))
+
+        sanitized = {}
+        for key, value in weights.items():
+            if not key.startswith("language_model."):
+                key = "language_model." + key
+            sanitized[key] = value
+        return sanitized
+
+    @property
+    def layers(self):
+        return self.language_model.model.layers

tests/test_models.py

@@ -1760,6 +1760,25 @@ def test_all_models(self):
                 "num_hidden_layers": 4,
                 "vocab_size": 1000,
             },
+            {
+                "model_type": "qwen3_vl",
+                "text_config": {
+                    "model_type": "qwen3",
+                    "hidden_size": 128,
+                    "num_hidden_layers": 4,
+                    "intermediate_size": 256,
+                    "num_attention_heads": 4,
+                    "num_key_value_heads": 2,
+                    "rms_norm_eps": 1e-5,
+                    "vocab_size": 1000,
+                    "head_dim": 32,
+                    "max_position_embeddings": 1000,
+                    "tie_word_embeddings": False,
+                    "rope_theta": 1000,
+                },
+                "num_hidden_layers": 4,
+                "vocab_size": 1000,
+            },
             {
                 "model_type": "seed_oss",
                 "hidden_size": 128,