Merge pull request #68 from shcherbak-ai/dev

Dev

Author: SergiiShcherbak

CHANGELOG.md

@@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 - **Refactor**: Code reorganization that doesn't change functionality but improves structure or maintainability
 
+## [0.17.0](https://github.com/shcherbak-ai/contextgem/releases/tag/v0.17.0) - 2025-08-24
+### Added
+- Multimodal LLM roles (`"extractor_multimodal"` and `"reasoner_multimodal"`) to support extraction of multimodal document-level concepts from both text and images. Previously, only text and vision roles were supported, requiring choosing either text or image context for extraction, not both.
+
 ## [0.16.1](https://github.com/shcherbak-ai/contextgem/releases/tag/v0.16.1) - 2025-08-19
 ### Fixed
 - Added support for `"minimal"` reasoning effort for gpt-5 models.

contextgem/__init__.py

@@ -20,7 +20,7 @@
 ContextGem - Effortless LLM extraction from documents
 """
 
-__version__ = "0.16.1"
+__version__ = "0.17.0"
 __author__ = "Shcherbak AI AS"
 
 from contextgem.public import (

contextgem/internal/base/aspects.py

@@ -88,7 +88,7 @@ class _Aspect(
         default_factory=list,
         description=(
             "Concepts associated with this aspect. Must be unique by name and description. "
-            "Concepts with vision LLM roles are not allowed."
+            "Concepts with vision or multimodal LLM roles are not allowed."
         ),
     )  # using Sequence field with list validator for type checking
     llm_role: LLMRoleAspect = Field(
@@ -154,16 +154,19 @@ def _validate_concepts_in_aspect(cls, concepts: list[_Concept]) -> list[_Concept
         :type concepts: list[_Concept]
         :raises ValueError: If multiple concepts have the same name.
         :raises ValueError: If multiple concepts have the same description.
-        :raises ValueError: If any concept has an LLM role ending with '_vision'.
+        :raises ValueError: If any concept has an LLM role ending with '_vision'
+            or '_multimodal'.
         :return: The validated list of '_Concept' instances.
         :rtype: list[_Concept]
         """
 
-        if concepts and any(i.llm_role.endswith("_vision") for i in concepts):
+        if concepts and any(
+            i.llm_role.endswith(x) for x in ("_vision", "_multimodal") for i in concepts
+        ):
             # Validate for Aspect-specific constraints.
             raise ValueError(
-                "Aspect concepts extraction using vision LLMs is not supported. "
-                "Vision LLMs can be used only for document concept extraction."
+                "Aspect-level concepts extraction using vision/multimodal LLMs is not supported. "
+                "Vision/multimodal LLMs can be used only for document-level concept extraction."
             )
         return concepts
 

contextgem/internal/base/concepts.py

@@ -98,7 +98,7 @@ class _Concept(_ExtractedItemsAttributeProcessor):
         default="extractor_text",
         description=(
             "LLM role used for this concept. Valid values: 'extractor_text', 'reasoner_text', "
-            "'extractor_vision', 'reasoner_vision'."
+            "'extractor_vision', 'reasoner_vision', 'extractor_multimodal', 'reasoner_multimodal'."
         ),
     )
     add_references: StrictBool = Field(

contextgem/internal/base/llms.py

@@ -118,7 +118,7 @@ class _LLMUsageOutputContainer(BaseModel):
     :vartype model: str
     :ivar role: The role of the model, which must be one of
         "extractor_text", "reasoner_text", "extractor_vision",
-        or "reasoner_vision".
+        "reasoner_vision", "extractor_multimodal", "reasoner_multimodal".
     :vartype role: LLMRoleAny
     :ivar is_fallback: Indicates whether the LLM is a fallback model.
     :vartype is_fallback: StrictBool
@@ -166,7 +166,7 @@ class _LLMCostOutputContainer(BaseModel):
     :vartype model: str
     :ivar role: The role of the model in processing, which can be one of:
                 "extractor_text", "reasoner_text", "extractor_vision",
-                "reasoner_vision".
+                "reasoner_vision", "extractor_multimodal", "reasoner_multimodal".
     :vartype role: LLMRoleAny
     :ivar is_fallback: Indicates if the LLM is a fallback model.
     :vartype is_fallback: bool

contextgem/internal/data_models.py

@@ -40,7 +40,12 @@
 ]
 
 LLMRoleAny = Literal[
-    "extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision"
+    "extractor_text",
+    "reasoner_text",
+    "extractor_vision",
+    "reasoner_vision",
+    "extractor_multimodal",
+    "reasoner_multimodal",
 ]
 
 LLMRoleAspect = Literal["extractor_text", "reasoner_text"]

contextgem/internal/typings/aliases.py

@@ -512,8 +512,8 @@ def _suppress_litellm_warnings_context() -> Generator[None, None, None]:
 
 def _suppress_litellm_warnings(func: F) -> F:
     """
-    Suppresses warnings related to Pydantic and httpx deprecation and serialization
-    in litellm>1.71.1 (latest available version as of 2025-07-10)
+    A decorator that suppresses warnings related to Pydantic and httpx deprecation
+    and serialization in litellm>1.71.1 (latest available version as of 2025-07-10).
 
     This decorator wraps both synchronous and asynchronous functions to suppress
     Pydantic and httpx warnings that originate from litellm's internal usage.

contextgem/internal/utils.py

@@ -59,8 +59,8 @@ class StringConcept(_StringConcept):
     :ivar examples: Example strings illustrating the concept usage.
     :vartype examples: list[StringExample]
     :ivar llm_role: The role of the LLM responsible for extracting the concept
-        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision").
-        Defaults to "extractor_text".
+        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision",
+        "extractor_multimodal", "reasoner_multimodal"). Defaults to "extractor_text".
     :vartype llm_role: LLMRoleAny
     :ivar add_justifications: Whether to include justifications for extracted items.
     :vartype add_justifications: bool
@@ -103,8 +103,8 @@ class BooleanConcept(_BooleanConcept):
     :ivar description: A brief description of the concept (non-empty string, stripped).
     :vartype description: str
     :ivar llm_role: The role of the LLM responsible for extracting the concept
-        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision").
-        Defaults to "extractor_text".
+        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision",
+        "extractor_multimodal", "reasoner_multimodal"). Defaults to "extractor_text".
     :vartype llm_role: LLMRoleAny
     :ivar add_justifications: Whether to include justifications for extracted items.
     :vartype add_justifications: bool
@@ -151,8 +151,8 @@ class NumericalConcept(_NumericalConcept):
         Defaults to "any" for auto-detection.
     :vartype numeric_type: Literal["int", "float", "any"]
     :ivar llm_role: The role of the LLM responsible for extracting the concept
-        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision").
-        Defaults to "extractor_text".
+        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision",
+        "extractor_multimodal", "reasoner_multimodal"). Defaults to "extractor_text".
     :vartype llm_role: LLMRoleAny
     :ivar add_justifications: Whether to include justifications for extracted items.
     :vartype add_justifications: bool
@@ -199,8 +199,8 @@ class RatingConcept(_RatingConcept):
         object (deprecated, will be removed in v1.0.0) or a tuple of (start, end) integers.
     :vartype rating_scale: RatingScale | tuple[int, int]
     :ivar llm_role: The role of the LLM responsible for extracting the concept
-        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision").
-        Defaults to "extractor_text".
+        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision",
+        "extractor_multimodal", "reasoner_multimodal"). Defaults to "extractor_text".
     :vartype llm_role: LLMRoleAny
     :ivar add_justifications: Whether to include justifications for extracted items.
     :vartype add_justifications: bool
@@ -277,8 +277,8 @@ class JsonObjectConcept(_JsonObjectConcept):
     :ivar examples: Example JSON objects illustrating the concept usage.
     :vartype examples: list[JsonObjectExample]
     :ivar llm_role: The role of the LLM responsible for extracting the concept
-        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision").
-        Defaults to "extractor_text".
+        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision",
+        "extractor_multimodal", "reasoner_multimodal"). Defaults to "extractor_text".
     :vartype llm_role: LLMRoleAny
     :ivar add_justifications: Whether to include justifications for extracted items.
     :vartype add_justifications: bool
@@ -322,8 +322,8 @@ class DateConcept(_DateConcept):
     :ivar description: A brief description of the concept (non-empty string, stripped).
     :vartype description: str
     :ivar llm_role: The role of the LLM responsible for extracting the concept
-        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision").
-        Defaults to "extractor_text".
+        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision",
+        "extractor_multimodal", "reasoner_multimodal"). Defaults to "extractor_text".
     :vartype llm_role: LLMRoleAny
     :ivar add_justifications: Whether to include justifications for extracted items.
     :vartype add_justifications: bool
@@ -383,8 +383,8 @@ class LabelConcept(_LabelConcept):
         "multi_label" for multiple label selection. Defaults to "multi_class".
     :vartype classification_type: ClassificationType
     :ivar llm_role: The role of the LLM responsible for extracting the concept
-        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision").
-        Defaults to "extractor_text".
+        ("extractor_text", "reasoner_text", "extractor_vision", "reasoner_vision",
+        "extractor_multimodal", "reasoner_multimodal"). Defaults to "extractor_text".
     :vartype llm_role: LLMRoleAny
     :ivar add_justifications: Whether to include justifications for extracted items.
     :vartype add_justifications: bool

contextgem/public/concepts.py

@@ -53,8 +53,9 @@ class Image(_Image):
             An image must be attached to a document. A document can have multiple images.
 
         - Extraction types:
-            Only concept extraction is supported for images. Use LLM with role ``"extractor_vision"``
-            or ``"reasoner_vision"`` to extract concepts from images.
+            Only document-level concept extraction is supported for images. Use LLM with role
+            ``"extractor_vision"``, ``"reasoner_vision"``, ``"extractor_multimodal"``,
+            or ``"reasoner_multimodal"`` to extract concepts from images.
 
     Example:
         .. literalinclude:: ../../../dev/usage_examples/docstrings/images/def_image.py