Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
1bf7f07
Update ModelCostData.java
vincentkoc Nov 7, 2025
3240de3
feat: BE support online eval with video
vincentkoc Nov 7, 2025
3c6e831
Update ModelCapabilitiesTest.java
vincentkoc Nov 7, 2025
7f075ec
Update MessageContentNormalizerTest.java
vincentkoc Nov 7, 2025
2af6e18
feat: BE llm message type for video
vincentkoc Nov 7, 2025
620cb49
chore: type for images in FE
vincentkoc Nov 7, 2025
914764e
Update modelCapabilities.ts
vincentkoc Nov 7, 2025
0521e58
feat: FE llm video type
vincentkoc Nov 7, 2025
40765d9
Update useMessageContent.ts
vincentkoc Nov 7, 2025
1e4e7eb
chore: media on prompt improvement panel
vincentkoc Nov 7, 2025
248d7e2
feat: video on prompts page
vincentkoc Nov 7, 2025
fcd3f62
feat: video on playground
vincentkoc Nov 7, 2025
1895068
chore: video datasets
vincentkoc Nov 7, 2025
fee3276
chore: video on experiments
vincentkoc Nov 7, 2025
1919f1e
feat: attachments for video
vincentkoc Nov 7, 2025
03276d5
feat: judge using video
vincentkoc Nov 7, 2025
405c099
Update chat_content_renderer_registry.py
vincentkoc Nov 7, 2025
63e3715
Update chat_prompt_template.py
vincentkoc Nov 7, 2025
dcbdf97
Update types.py
vincentkoc Nov 7, 2025
7e82ba9
Update evaluator.py
vincentkoc Nov 7, 2025
e56867b
feat: SDK model capabilities
vincentkoc Nov 7, 2025
6420e9f
Update OnlineScoringEngine.java
vincentkoc Nov 7, 2025
6ab3021
Update MessageContentNormalizer.java
vincentkoc Nov 7, 2025
d059f51
chore: supports vision is mapped
vincentkoc Nov 7, 2025
2eb230a
Merge branch 'main' into feat/video-eval
vincentkoc Nov 7, 2025
8b2c358
Update MessageContentNormalizer.java
vincentkoc Nov 7, 2025
5cd8534
chore: lint
vincentkoc Nov 7, 2025
2d493d0
Merge branch 'feat/video-eval' of https://github.com/comet-ml/opik in…
vincentkoc Nov 7, 2025
a48c229
Update model_capabilities.py
vincentkoc Nov 7, 2025
4fdbf20
Update apps/opik-frontend/src/lib/modelCapabilities.ts
vincentkoc Nov 7, 2025
081f456
chore: copilot fixes
vincentkoc Nov 7, 2025
0d6ab81
Merge branch 'feat/video-eval' of https://github.com/comet-ml/opik in…
vincentkoc Nov 7, 2025
6af3039
Update evaluate_multimodal.mdx
vincentkoc Nov 7, 2025
e0501a6
chore: reset python sdk
vincentkoc Nov 8, 2025
01f21bf
Merge branch 'main' into feat/video-eval
vincentkoc Nov 8, 2025
a47b41f
fixing base64 video in playground & dataset
YarivHashaiComet Nov 10, 2025
d0cd62a
fixing base64 img & video on annodation queue
YarivHashaiComet Nov 10, 2025
6048173
removing unrelated code
YarivHashaiComet Nov 10, 2025
c4757c5
Merge branch 'main' into feat/video-eval
vincentkoc Nov 11, 2025
57b718d
[OPIK-3030] [BE] Cursor fixes for videos in online scoring (#4025)
AndreiCautisanu Nov 11, 2025
c34303c
Merge branch 'main' into feat/video-eval
vincentkoc Nov 11, 2025
8bc5878
[OPIK-3040] add video thumbnail preview
CometActions Nov 12, 2025
0c6db42
[OPIK-3040] add file input for video/image
CometActions Nov 12, 2025
d9d6280
[OPIK-3040] merge main
CometActions Nov 12, 2025
10a5179
fix: revert breaking change
vincentkoc Nov 12, 2025
c8f97df
fix: video model mapping
vincentkoc Nov 12, 2025
05319f1
Update OnlineScoringEngine.java
vincentkoc Nov 12, 2025
48e6e2f
Merge branch 'main' into feat/video-eval
vincentkoc Nov 12, 2025
082bcd1
fix: SDK issue with null values in template
vincentkoc Nov 12, 2025
bc421bb
Merge branch 'feat/video-eval' of https://github.com/comet-ml/opik in…
vincentkoc Nov 12, 2025
bc18f3b
Update test_prompt_template.py
vincentkoc Nov 12, 2025
cb38c8d
[OPIK-3040] fix formats
CometActions Nov 13, 2025
c41cedf
[OPIK-3040] handle base64 as file
CometActions Nov 13, 2025
3807c0c
Merge branch 'feat/video-eval' of https://github.com/comet-ml/opik in…
CometActions Nov 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,6 @@ public record ModelCostData(String litellmProvider,
String outputCostPerToken,
String cacheCreationInputTokenCost,
String cacheReadInputTokenCost,
boolean supportsVision) {
boolean supportsVision,
boolean supportsVideoInput) {
}

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ protected void score(@NonNull TraceToScoreLlmAsJudge message) {
String modelName = message.llmAsJudgeCode().model().name();
var llmProvider = llmProviderFactory.getLlmProvider(modelName);
var strategy = StructuredOutputStrategy.getStrategy(llmProvider, modelName);
scoreRequest = OnlineScoringEngine.prepareLlmRequest(message.llmAsJudgeCode(), trace, strategy);
scoreRequest = OnlineScoringEngine.prepareLlmRequest(message.llmAsJudgeCode(), trace, strategy,
llmProvider);
} catch (Exception exception) {
userFacingLogger.error("Error preparing LLM request for traceId '{}': \n\n{}",
trace.id(), exception.getMessage());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ private void processScoring(TraceThreadToScoreLlmAsJudge message, List<Trace> tr
String modelName = message.code().model().name();
var llmProvider = llmProviderFactory.getLlmProvider(modelName);
var strategy = StructuredOutputStrategy.getStrategy(llmProvider, modelName);
scoreRequest = OnlineScoringEngine.prepareThreadLlmRequest(message.code(), traces, strategy);
scoreRequest = OnlineScoringEngine.prepareThreadLlmRequest(message.code(), traces, strategy, llmProvider);
} catch (Exception exception) {
userFacingLogger.error("Error preparing LLM request for threadId: '{}': \n\n{}",
threadId, exception.getMessage());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,10 +185,29 @@ static Map<String, String> getOrDefault(Map<String, JsonNode> data) {
.stream()
.map(Map::entrySet)
.flatMap(Collection::stream)
.map(entry -> Map.entry(entry.getKey(), entry.getValue().toString()))
.map(entry -> Map.entry(entry.getKey(), jsonNodeToString(entry.getValue())))
.collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
}

/**
* Converts a JsonNode to its string representation without double-escaping.
* For TextNodes, returns the raw text value. For other node types, serializes to JSON.
*
* @param node The JsonNode to convert
* @return The string representation
*/
private static String jsonNodeToString(JsonNode node) {
if (node == null || node.isNull()) {
return "";
}
// For text nodes, extract the raw text value without additional JSON serialization
if (node.isTextual()) {
return node.asText();
}
// For other node types (objects, arrays, numbers, booleans), serialize to JSON
return JsonUtils.writeValueAsString(node);
}

static String getOrDefault(UUID value) {
return Optional.ofNullable(value).map(UUID::toString).orElse("");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ public void createAndStreamResponse(
public ChatResponse scoreTrace(@NonNull ChatRequest chatRequest,
@NonNull LlmAsJudgeModelParameters modelParameters,
@NonNull String workspaceId) {
// Standard path for non-multimodal or non-custom-LLM providers
var languageModelClient = llmProviderFactory.getLanguageModel(workspaceId, modelParameters);

ChatResponse chatResponse;
Expand Down
Loading
Loading