Skip to content

test: ensure comprehensive error code coverage in quality module #101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hozi8-web3 wants to merge 12 commits into
base: main
Choose a base branch
from
Open

test: ensure comprehensive error code coverage in quality module #101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
0d9a8c7
test: achieve comprehensive error code coverage for quality checks
hozi8-web3 Feb 20, 2026
1c062ad
test: achieve comprehensive error code coverage for quality checks
hozi8-web3 Feb 21, 2026
8db5309
test: fix VF07 trigger condition and remove stale comment
hozi8-web3 Feb 21, 2026
d26aa21
test: fix flaky BV15 and wrong assertion in LV11 test
hozi8-web3 Feb 21, 2026
534c3db
Merge branch 'ObjectiveAI:main' into main
hozi8-web3 Feb 22, 2026
8832d72
test(functions:quality): fix tests after main branch divergence
hozi8-web3 Feb 22, 2026
0cd1e1a
test(functions:quality): fix remaining failing assertions
hozi8-web3 Feb 22, 2026
775355b
test(functions:quality): fix flaky VF24 array max_items assertion
hozi8-web3 Feb 22, 2026
8c6ebeb
feat(ui): implement premium skeleton loaders for web dashboard
hozi8-web3 Feb 22, 2026
8bcd452
Merge branch 'ObjectiveAI:main' into main
hozi8-web3 Feb 22, 2026
9c50d98
Merge branch 'main' into main
hozi8-web3 Feb 23, 2026
8915e0d
Merge branch 'ObjectiveAI:main' into main
hozi8-web3 Feb 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 50 additions & 2 deletions objectiveai-rs/src/functions/quality/check_branch_scalar_function_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@ use crate::chat::completions::request::{
RichContentExpression, RichContentPartExpression, UserMessageExpression,
};
use crate::functions::expression::{
ArrayInputSchema, BooleanInputSchema, Expression, InputSchema,
IntegerInputSchema, ObjectInputSchema, StringInputSchema, WithExpression,
AnyOfInputSchema, ArrayInputSchema, BooleanInputSchema, Expression,
InputSchema, IntegerInputSchema, ObjectInputSchema, StringInputSchema,
WithExpression,
};
use crate::functions::quality::check_branch_scalar_function;
use crate::functions::{Remote,
Expand Down Expand Up @@ -1134,3 +1135,50 @@ fn all_tasks_skipped() {
};
test_err(&f, "CV42");
}
#[test]
fn no_example_inputs() {
let f = RemoteFunction::Scalar {
description: "test".to_string(),
input_schema: InputSchema::AnyOf(AnyOfInputSchema { any_of: vec![] }),
input_maps: None,
tasks: vec![TaskExpression::ScalarFunction(
ScalarFunctionTaskExpression {
owner: "test".to_string(),
repository: "test".to_string(),
commit: "abc123".to_string(),
skip: None,
map: None,
input: WithExpression::Expression(Expression::Starlark(
"input".to_string(),
)),
output: Expression::Starlark("output".to_string()),
},
)],
};
test_err(&f, "QI01");
}

#[test]
fn placeholder_scalar_field_validation_fails() {
let f = RemoteFunction::Scalar {
description: "test".to_string(),
input_schema: InputSchema::Integer(IntegerInputSchema {
description: None,
minimum: Some(1),
maximum: Some(10),
}),
input_maps: None,
tasks: vec![TaskExpression::PlaceholderScalarFunction(
PlaceholderScalarFunctionTaskExpression {
input_schema: InputSchema::AnyOf(AnyOfInputSchema { any_of: vec![] }),
skip: None,
map: None,
input: WithExpression::Expression(Expression::Starlark(
"input".to_string(),
)),
output: Expression::Starlark("output".to_string()),
},
)],
};
test_err(&f, "CV04");
}
262 changes: 261 additions & 1 deletion objectiveai-rs/src/functions/quality/check_branch_vector_function_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use crate::chat::completions::request::{
UserMessageExpression,
};
use crate::functions::expression::{
ArrayInputSchema, BooleanInputSchema, Expression, InputMaps, InputSchema,
AnyOfInputSchema, ArrayInputSchema, BooleanInputSchema, Expression, InputMaps, InputSchema,
IntegerInputSchema, ObjectInputSchema, StringInputSchema, WithExpression,
};
use crate::functions::quality::check_branch_vector_function;
Expand Down Expand Up @@ -2218,3 +2218,263 @@ fn output_length_less_than_2() {
};
test_err(&f, "VF03");
}
#[test]
fn input_maps_compilation_fails() {
let f = RemoteFunction::Vector {
description: "test".to_string(),
input_schema: InputSchema::Object(ObjectInputSchema {
description: None,
properties: index_map! {
"items" => InputSchema::Array(ArrayInputSchema {
description: None,
min_items: Some(2),
max_items: Some(2),
items: Box::new(InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})),
}),
"label" => InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})
},
required: Some(vec!["items".to_string(), "label".to_string()]),
}),
input_maps: Some(InputMaps::Many(vec![
Expression::Starlark("1 + ".to_string()), // Syntax error
])),
tasks: vec![TaskExpression::ScalarFunction(ScalarFunctionTaskExpression {
owner: "test".to_string(),
repository: "test".to_string(),
commit: "abc123".to_string(),
skip: None,
map: Some(0),
input: WithExpression::Expression(Expression::Starlark("map".to_string())),
output: Expression::Starlark("output".to_string()),
})],
output_length: WithExpression::Expression(Expression::Starlark("len(input['items'])".to_string())),
input_split: WithExpression::Expression(Expression::Starlark("[{'items': [x], 'label': input['label']} for x in input['items']]".to_string())),
input_merge: WithExpression::Expression(Expression::Starlark("{'items': [x['items'][0] for x in input], 'label': input[0]['label']}".to_string())),
};
test_err(&f, "BV08");
}

#[test]
fn input_merge_fails_on_subset() {
let f = RemoteFunction::Vector {
description: "test".to_string(),
input_schema: InputSchema::Object(ObjectInputSchema {
description: None,
properties: index_map! {
"items" => InputSchema::Array(ArrayInputSchema {
description: None,
min_items: Some(3),
max_items: Some(3),
items: Box::new(InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})),
}),
"label" => InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})
},
required: Some(vec!["items".to_string(), "label".to_string()]),
}),
input_maps: None,
tasks: vec![TaskExpression::VectorFunction(VectorFunctionTaskExpression {
owner: "test".to_string(),
repository: "test".to_string(),
commit: "abc123".to_string(),
skip: None,
map: None,
input: WithExpression::Expression(Expression::Starlark("input".to_string())),
output: Expression::Starlark("output".to_string()),
})],
output_length: WithExpression::Expression(Expression::Starlark("len(input['items'])".to_string())),
input_split: WithExpression::Expression(Expression::Starlark("[{'items': [x], 'label': input['label']} for x in input['items']]".to_string())),
input_merge: WithExpression::Expression(Expression::Starlark("{'items': [x['items'][0] for x in input], 'label': 1/0}".to_string())),
};
test_err(&f, "VF10");
}

#[test]
fn no_example_inputs() {
let f = RemoteFunction::Vector {
description: "test".to_string(),
input_schema: InputSchema::AnyOf(AnyOfInputSchema { any_of: vec![] }),
input_maps: None,
tasks: vec![TaskExpression::VectorFunction(VectorFunctionTaskExpression {
owner: "test".to_string(),
repository: "test".to_string(),
commit: "abc123".to_string(),
skip: None,
map: None,
input: WithExpression::Expression(Expression::Starlark("input".to_string())),
output: Expression::Starlark("output".to_string()),
})],
output_length: WithExpression::Expression(Expression::Starlark("1".to_string())),
input_split: WithExpression::Expression(Expression::Starlark("[input]".to_string())),
input_merge: WithExpression::Expression(Expression::Starlark("input[0]".to_string())),
};
test_err(&f, "QI01");
}

#[test]
fn fixed_mapped_input() {
let f = RemoteFunction::Vector {
description: "test".to_string(),
input_schema: InputSchema::Object(ObjectInputSchema {
description: None,
properties: index_map! {
"items" => InputSchema::Array(ArrayInputSchema {
description: None,
min_items: Some(2),
max_items: Some(2),
items: Box::new(InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})),
}),
"label" => InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})
},
required: Some(vec!["items".to_string(), "label".to_string()]),
}),
input_maps: Some(InputMaps::Many(vec![
Expression::Starlark("['fixed_val', 'fixed_val']".to_string()),
])),
tasks: vec![TaskExpression::ScalarFunction(ScalarFunctionTaskExpression {
owner: "test".to_string(),
repository: "test".to_string(),
commit: "abc123".to_string(),
skip: None,
map: Some(0),
input: WithExpression::Expression(Expression::Starlark("map".to_string())),
output: Expression::Starlark("output".to_string()),
})],
output_length: WithExpression::Expression(Expression::Starlark("len(input['items'])".to_string())),
input_split: WithExpression::Expression(Expression::Starlark("[{'items': [x], 'label': input['label']} for x in input['items']]".to_string())),
input_merge: WithExpression::Expression(Expression::Starlark("{'items': [x['items'][0] for x in input], 'label': input[0]['label']}".to_string())),
};
test_err(&f, "BV08");
}

#[test]
fn all_mapped_inputs_equal() {
let f = RemoteFunction::Vector {
description: "test".to_string(),
input_schema: InputSchema::Object(ObjectInputSchema {
description: None,
properties: index_map! {
"items" => InputSchema::Array(ArrayInputSchema {
description: None,
min_items: Some(2),
max_items: Some(2),
items: Box::new(InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})),
}),
"label" => InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})
},
required: Some(vec!["items".to_string(), "label".to_string()]),
}),
input_maps: Some(InputMaps::Many(vec![
Expression::Starlark("[input['label'], input['label']]".to_string()),
])),
tasks: vec![TaskExpression::ScalarFunction(ScalarFunctionTaskExpression {
owner: "test".to_string(),
repository: "test".to_string(),
commit: "abc123".to_string(),
skip: None,
map: Some(0),
input: WithExpression::Expression(Expression::Starlark("map".to_string())),
output: Expression::Starlark("output".to_string()),
})],
output_length: WithExpression::Expression(Expression::Starlark("len(input['items'])".to_string())),
input_split: WithExpression::Expression(Expression::Starlark("[{'items': [x], 'label': input['label']} for x in input['items']]".to_string())),
input_merge: WithExpression::Expression(Expression::Starlark("{'items': [x['items'][0] for x in input], 'label': input[0]['label']}".to_string())),
};
test_err(&f, "BV08");
}

#[test]
fn placeholder_scalar_field_fails() {
let f = RemoteFunction::Vector {
description: "test".to_string(),
input_schema: InputSchema::Object(ObjectInputSchema {
description: None,
properties: index_map! {
"items" => InputSchema::Array(ArrayInputSchema {
description: None,
min_items: Some(2),
max_items: Some(2),
items: Box::new(InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})),
})
},
required: Some(vec!["items".to_string()]),
}),
input_maps: Some(InputMaps::Many(vec![
Expression::Starlark("input['items']".to_string()),
])),
tasks: vec![TaskExpression::PlaceholderScalarFunction(PlaceholderScalarFunctionTaskExpression {
input_schema: InputSchema::AnyOf(AnyOfInputSchema { any_of: vec![] }),
skip: None,
map: Some(0),
input: WithExpression::Expression(Expression::Starlark("map".to_string())),
output: Expression::Starlark("output".to_string()),
})],
output_length: WithExpression::Expression(Expression::Starlark("len(input['items'])".to_string())),
input_split: WithExpression::Expression(Expression::Starlark("[{'items': [x]} for x in input['items']]".to_string())),
input_merge: WithExpression::Expression(Expression::Starlark("{'items': [x['items'][0] for x in input]}".to_string())),
};
test_err(&f, "BV08");
}

#[test]
fn placeholder_vector_field_fails() {
let f = RemoteFunction::Vector {
description: "test".to_string(),
input_schema: InputSchema::Object(ObjectInputSchema {
description: None,
properties: index_map! {
"items" => InputSchema::Array(ArrayInputSchema {
description: None,
min_items: Some(2),
max_items: Some(2),
items: Box::new(InputSchema::String(StringInputSchema {
description: None,
r#enum: None,
})),
})
},
required: Some(vec!["items".to_string()]),
}),
input_maps: None,
tasks: vec![TaskExpression::PlaceholderVectorFunction(PlaceholderVectorFunctionTaskExpression {
input_schema: InputSchema::AnyOf(AnyOfInputSchema { any_of: vec![] }),
output_length: WithExpression::Expression(Expression::Starlark("len(input['items'])".to_string())),
input_split: WithExpression::Expression(Expression::Starlark("[{'items': [x]} for x in input['items']]".to_string())),
input_merge: WithExpression::Expression(Expression::Starlark("{'items': [x['items'][0] for x in input]}".to_string())),
skip: None,
map: None,
input: WithExpression::Expression(Expression::Starlark("input".to_string())),
output: Expression::Starlark("output".to_string()),
})],
output_length: WithExpression::Expression(Expression::Starlark("len(input['items'])".to_string())),
input_split: WithExpression::Expression(Expression::Starlark("[{'items': [x]} for x in input['items']]".to_string())),
input_merge: WithExpression::Expression(Expression::Starlark("{'items': [x['items'][0] for x in input]}".to_string())),
};
test_err(&f, "CV05");
}
52 changes: 49 additions & 3 deletions objectiveai-rs/src/functions/quality/check_leaf_scalar_function_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ use crate::chat::completions::request::{
SystemMessageExpression, ToolMessageExpression, UserMessageExpression,
};
use crate::functions::expression::{
ArrayInputSchema, BooleanInputSchema, Expression, ImageInputSchema,
InputSchema, IntegerInputSchema, ObjectInputSchema, StringInputSchema,
WithExpression,
AnyOfInputSchema, ArrayInputSchema, BooleanInputSchema, Expression,
ImageInputSchema, InputSchema, IntegerInputSchema, ObjectInputSchema,
StringInputSchema, WithExpression,
};
use crate::functions::quality::check_leaf_scalar_function;
use crate::functions::{Remote,
Expand Down Expand Up @@ -2313,3 +2313,49 @@ fn all_tasks_skipped() {
};
test_err(&f, "CV42");
}

#[test]
fn no_example_inputs() {
let f = RemoteFunction::Scalar {
description: "test".to_string(),
input_schema: InputSchema::AnyOf(AnyOfInputSchema { any_of: vec![] }),
input_maps: None,
tasks: vec![TaskExpression::VectorCompletion(
VectorCompletionTaskExpression {
skip: None,
map: None,
messages: WithExpression::Value(vec![WithExpression::Value(
MessageExpression::User(UserMessageExpression {
content: WithExpression::Value(
RichContentExpression::Parts(vec![
WithExpression::Value(
RichContentPartExpression::Text {
text: WithExpression::Value(
"Hello".to_string(),
),
},
),
]),
),
name: None,
}),
)]),
tools: None,
responses: WithExpression::Value(vec![
WithExpression::Value(RichContentExpression::Parts(vec![
WithExpression::Value(RichContentPartExpression::Text {
text: WithExpression::Value("Yes".to_string()),
}),
])),
WithExpression::Value(RichContentExpression::Parts(vec![
WithExpression::Value(RichContentPartExpression::Text {
text: WithExpression::Value("No".to_string()),
}),
])),
]),
output: Expression::Starlark("output['scores'][0]".to_string()),
},
)],
};
test_err(&f, "QI01");
}
Loading