[OPIK-2940] [FE][BE][DOCS] Video: Add support for Video LLM-as-a-judge, Datasets and Playground - WIP DO NOT MERGE

apps/opik-backend/src/main/java/com/comet/opik/api/ModelCostData.java

@@ -13,5 +13,6 @@ public record ModelCostData(String litellmProvider,
         String outputCostPerToken,
         String cacheCreationInputTokenCost,
         String cacheReadInputTokenCost,
-        boolean supportsVision) {
+        boolean supportsVision,
+        boolean supportsVideoInput) {
 }

apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/events/OnlineScoringEngine.java

@@ -5,8 +5,8 @@
 import com.comet.opik.api.Trace;
 import com.comet.opik.api.evaluators.LlmAsJudgeMessage;
 import com.comet.opik.domain.evaluators.python.TraceThreadPythonEvaluatorRequest;
-import com.comet.opik.domain.llm.MessageContentNormalizer;
 import com.comet.opik.domain.llm.structuredoutput.StructuredOutputStrategy;
+import com.comet.opik.utils.JsonUtils;
 import com.comet.opik.utils.TemplateParseUtils;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -15,10 +15,13 @@
 import com.google.api.gax.rpc.InvalidArgumentException;
 import com.jayway.jsonpath.JsonPath;
 import dev.langchain4j.data.message.ChatMessage;
+import dev.langchain4j.data.message.Content;
 import dev.langchain4j.data.message.ImageContent;
 import dev.langchain4j.data.message.SystemMessage;
 import dev.langchain4j.data.message.TextContent;
 import dev.langchain4j.data.message.UserMessage;
+import dev.langchain4j.data.message.VideoContent;
+import dev.langchain4j.data.video.Video;
 import dev.langchain4j.model.chat.request.ChatRequest;
 import dev.langchain4j.model.chat.response.ChatResponse;
 import jakarta.validation.constraints.NotNull;
@@ -60,9 +63,8 @@ public class OnlineScoringEngine {
 
     private static final Pattern JSON_BLOCK_PATTERN = Pattern.compile(
             "```(?:json)?\\s*(\\{.*?})\\s*```", Pattern.DOTALL);
-    private static final Pattern IMAGE_PLACEHOLDER_PATTERN = Pattern.compile(
-            Pattern.quote(MessageContentNormalizer.IMAGE_PLACEHOLDER_START) + "(.*?)"
-                    + Pattern.quote(MessageContentNormalizer.IMAGE_PLACEHOLDER_END),
+    private static final Pattern MEDIA_PLACEHOLDER_PATTERN = Pattern.compile(
+            "<<<(image|video)>>>(.*?)<<</(image|video)>>>",
             Pattern.DOTALL);
 
     /**
@@ -221,36 +223,42 @@ static List<MessageVariableMapping> toVariableMapping(Map<String, String> evalua
     }
 
     private UserMessage buildUserMessage(String content) {
-        var matcher = IMAGE_PLACEHOLDER_PATTERN.matcher(content);
-        var foundAny = false;
+        var matcher = MEDIA_PLACEHOLDER_PATTERN.matcher(content);
+
+        if (!matcher.find()) {
+            return UserMessage.from(content);
+        }
+
+        matcher.reset();
 
         var builder = UserMessage.builder();
         var lastIndex = 0;
 
         while (matcher.find()) {
-            foundAny = true;
-
             if (matcher.start() > lastIndex) {
                 var textSegment = content.substring(lastIndex, matcher.start());
                 appendTextContent(builder, textSegment);
             }
 
-            var url = matcher.group(1).trim();
-            if (!url.isEmpty()) {
-                // Unescape HTML entities for backward compatibility with templates using {{variable}}
-                // RECOMMENDED: Use {{{variable}}} (triple braces) or {{&variable}} in Mustache templates
-                // to prevent HTML escaping of URLs in the first place
-                var unescapedUrl = StringEscapeUtils.unescapeHtml4(url);
-                builder.addContent(ImageContent.from(unescapedUrl));
+            var mediaType = matcher.group(1);
+            var rawValue = matcher.group(2).trim();
+            var placeholderText = matcher.group(0);
+
+            if (!rawValue.isEmpty()) {
+                // Mustache templates can HTML-escape URLs; unescape before building structured content
+                // so providers receive the exact user-specified payload.
+                var unescapedValue = StringEscapeUtils.unescapeHtml4(rawValue);
+                var mediaContent = createMediaContent(mediaType, unescapedValue);
+                if (mediaContent != null) {
+                    builder.addContent(mediaContent);
+                } else {
+                    appendTextContent(builder, placeholderText);
+                }
             }
 
             lastIndex = matcher.end();
         }
 
-        if (!foundAny) {
-            return UserMessage.from(content);
-        }
-
         if (lastIndex < content.length()) {
             var trailingText = content.substring(lastIndex);
             appendTextContent(builder, trailingText);
@@ -259,12 +267,207 @@ private UserMessage buildUserMessage(String content) {
         return builder.build();
     }
 
+    private Content createMediaContent(String mediaType, String rawValue) {
+        if ("image".equalsIgnoreCase(mediaType)) {
+            return createImageContent(rawValue);
+        }
+
+        if ("video".equalsIgnoreCase(mediaType)) {
+            return createVideoContent(rawValue);
+        }
+
+        log.warn("Unsupported media type placeholder encountered: '{}'", mediaType);
+        return null;
+    }
+
     private void appendTextContent(UserMessage.Builder builder, String textSegment) {
         if (StringUtils.isNotBlank(textSegment)) {
             builder.addContent(TextContent.from(textSegment));
         }
     }
 
+    private Content createImageContent(String rawValue) {
+        if (StringUtils.isBlank(rawValue)) {
+            return null;
+        }
+
+        try {
+            return ImageContent.from(rawValue);
+        } catch (IllegalArgumentException exception) {
+            log.warn("Failed to build image content for placeholder: {}", rawValue, exception);
+            return null;
+        }
+    }
+
+    private Content createVideoContent(String rawValue) {
+        var trimmed = StringUtils.trimToEmpty(rawValue);
+        if (trimmed.isEmpty()) {
+            return null;
+        }
+
+        if (looksLikeJson(trimmed)) {
+            Object parsed = tryParseJson(trimmed);
+            var fromJson = convertVideoPayload(parsed);
+            if (fromJson != null) {
+                return fromJson;
+            }
+        }
+
+        return buildVideoFromValue(trimmed, null);
+    }
+
+    private Content convertVideoPayload(Object payload) {
+        if (payload instanceof Map<?, ?> mapPayload) {
+            return convertVideoObject(mapPayload);
+        }
+
+        if (payload instanceof List<?> listPayload) {
+            for (var entry : listPayload) {
+                var converted = convertVideoPayload(entry);
+                if (converted != null) {
+                    return converted;
+                }
+            }
+        }
+
+        if (payload instanceof String stringPayload) {
+            return buildVideoFromValue(stringPayload, null);
+        }
+
+        return null;
+    }
+
+    private Content convertVideoObject(Map<?, ?> payload) {
+        var type = stringValue(payload.get("type"));
+
+        if ("video_url".equalsIgnoreCase(type) || payload.containsKey("video_url")) {
+            return buildVideoFromVideoUrl(payload.get("video_url"));
+        }
+
+        if ("file".equalsIgnoreCase(type) || payload.containsKey("file")) {
+            return buildVideoFromFile(payload.get("file"));
+        }
+
+        if (type == null) {
+            return buildVideoFromVideoUrl(payload.get("url"));
+        }
+
+        return null;
+    }
+
+    private Content buildVideoFromVideoUrl(Object videoNode) {
+        if (videoNode instanceof String url) {
+            return buildVideoFromValue(url, null);
+        }
+
+        if (videoNode instanceof Map<?, ?> videoMap) {
+            var url = stringValue(videoMap.get("url"));
+            var mimeType = firstNonBlank(
+                    stringValue(videoMap.get("mime_type")),
+                    stringValue(videoMap.get("format")));
+            return buildVideoFromValue(url, mimeType);
+        }
+
+        return null;
+    }
+
+    private Content buildVideoFromFile(Object fileNode) {
+        if (!(fileNode instanceof Map<?, ?> fileMap)) {
+            return null;
+        }
+
+        var fileId = stringValue(fileMap.get("file_id"));
+        if (StringUtils.isNotBlank(fileId)) {
+            return buildVideoFromValue(fileId, stringValue(fileMap.get("format")));
+        }
+
+        var fileData = stringValue(fileMap.get("file_data"));
+        if (StringUtils.isNotBlank(fileData)) {
+            return buildVideoFromBase64(fileData, stringValue(fileMap.get("format")));
+        }
+
+        return null;
+    }
+
+    private Content buildVideoFromValue(String value, String mimeType) {
+        if (StringUtils.isBlank(value)) {
+            return null;
+        }
+
+        if (value.startsWith("data:video/")) {
+            return buildVideoFromUrl(value, mimeType);
+        }
+
+        if (value.startsWith("http://") || value.startsWith("https://")) {
+            return buildVideoFromUrl(value, mimeType);
+        }
+
+        return buildVideoFromBase64(value, mimeType);
+    }
+
+    private Content buildVideoFromUrl(String url, String mimeType) {
+        if (StringUtils.isBlank(url)) {
+            return null;
+        }
+
+        try {
+            if (StringUtils.isBlank(mimeType)) {
+                return VideoContent.from(url);
+            }
+
+            var video = Video.builder()
+                    .url(url)
+                    .mimeType(mimeType)
+                    .build();
+            return VideoContent.from(video);
+        } catch (IllegalArgumentException exception) {
+            log.warn("Failed to build video content for url placeholder: {}", url, exception);
+            return null;
+        }
+    }
+
+    private Content buildVideoFromBase64(String base64Data, String mimeType) {
+        if (StringUtils.isBlank(base64Data)) {
+            return null;
+        }
+
+        var safeMimeType = StringUtils.defaultIfBlank(mimeType, "video/mp4");
+        try {
+            return VideoContent.from(base64Data, safeMimeType);
+        } catch (IllegalArgumentException exception) {
+            log.warn("Failed to build video content from base64 payload", exception);
+            return null;
+        }
+    }
+
+    private boolean looksLikeJson(String value) {
+        var trimmed = StringUtils.trimToEmpty(value);
+        return trimmed.startsWith("{") || trimmed.startsWith("[");
+    }
+
+    private Object tryParseJson(String rawValue) {
+        try {
+            return JsonUtils.readValue(rawValue, Object.class);
+        } catch (RuntimeException exception) {
+            log.warn("Failed to parse media placeholder as JSON: {}", rawValue, exception);
+            return null;
+        }
+    }
+
+    private String stringValue(Object value) {
+        return value instanceof String str ? str : null;
+    }
+
+    private String firstNonBlank(String first, String second) {
+        if (StringUtils.isNotBlank(first)) {
+            return first;
+        }
+        if (StringUtils.isNotBlank(second)) {
+            return second;
+        }
+        return null;
+    }
+
     private static String extractFromJson(JsonNode json, String path) {
         Map<String, Object> forcedObject;
         try {