Skip to content

chore(ai): Clean up model costs v1 #4951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 18, 2025
+12 −158
Merged

chore(ai): Clean up model costs v1 #4951

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions relay-event-normalization/src/event.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2293,7 +2293,6 @@ mod tests {
&NormalizationConfig {
ai_model_costs: Some(&ModelCosts {
version: 2,
costs: vec![],
models: HashMap::from([
(
"claude-2.1".to_owned(),
Expand Down Expand Up @@ -2399,7 +2398,6 @@ mod tests {
&NormalizationConfig {
ai_model_costs: Some(&ModelCosts {
version: 2,
costs: vec![],
models: HashMap::from([
(
"claude-2.1".to_owned(),
Expand Down Expand Up @@ -2496,7 +2494,6 @@ mod tests {
&NormalizationConfig {
ai_model_costs: Some(&ModelCosts {
version: 2,
costs: vec![],
models: HashMap::from([(
"claude-2.1".to_owned(),
ModelCostV2 {
Expand Down Expand Up @@ -2580,7 +2577,6 @@ mod tests {
&NormalizationConfig {
ai_model_costs: Some(&ModelCosts {
version: 2,
costs: vec![],
models: HashMap::from([
(
"claude-2.1".to_owned(),
Expand Down Expand Up @@ -2660,7 +2656,6 @@ mod tests {
&NormalizationConfig {
ai_model_costs: Some(&ModelCosts {
version: 2,
costs: vec![],
models: HashMap::new(),
}),
..NormalizationConfig::default()
Expand Down Expand Up @@ -2704,7 +2699,6 @@ mod tests {
&NormalizationConfig {
ai_model_costs: Some(&ModelCosts {
version: 2,
costs: vec![],
models: HashMap::new(),
}),
..NormalizationConfig::default()
Expand Down
164 changes: 12 additions & 152 deletions relay-event-normalization/src/normalize/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ use std::collections::HashMap;
use std::hash::Hash;

use relay_base_schema::metrics::MetricUnit;
use relay_common::glob2::LazyGlob;
use relay_event_schema::protocol::{Event, VALID_PLATFORMS};
use relay_protocol::{FiniteF64, RuleCondition};
use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -219,25 +218,10 @@ pub struct PerformanceScoreConfig {

/// A mapping of AI model types (like GPT-4) to their respective costs.
///
/// This struct supports multiple versions with different cost structures:
/// - Version 1: Array-based costs with glob pattern matching for model IDs (uses `costs` field)
/// - Version 2: Dictionary-based costs with exact model ID keys and granular token pricing (uses `models` field)
/// This struct uses a dictionary-based cost structure with exact model ID keys and granular
/// token pricing.
///
/// Example V1 JSON:
/// ```json
/// {
/// "version": 1,
/// "costs": [
/// {
/// "modelId": "gpt-4*",
/// "forCompletion": false,
/// "costPer1kTokens": 0.03
/// }
/// ]
/// }
/// ```
///
/// Example V2 JSON:
/// Example JSON:
/// ```json
/// {
/// "version": 2,
Expand All @@ -257,72 +241,35 @@ pub struct ModelCosts {
/// The version of the model cost struct
pub version: u16,

/// The mappings of model ID => cost (used in version 1)
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub costs: Vec<ModelCost>,

/// The mappings of model ID => cost as a dictionary (version 2)
/// The mappings of model ID => cost as a dictionary
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
pub models: HashMap<String, ModelCostV2>,
}

impl ModelCosts {
const MAX_SUPPORTED_VERSION: u16 = 2;
const SUPPORTED_VERSION: u16 = 2;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Currently, only 1 version is supported


/// `true` if the model costs are empty and the version is supported.
pub fn is_empty(&self) -> bool {
(self.costs.is_empty() && self.models.is_empty()) || !self.is_enabled()
self.models.is_empty() || !self.is_enabled()
}

/// `false` if measurement and metrics extraction should be skipped.
pub fn is_enabled(&self) -> bool {
self.version > 0 && self.version <= ModelCosts::MAX_SUPPORTED_VERSION
self.version == Self::SUPPORTED_VERSION
}

/// Gets the cost per token, if defined for the given model.
pub fn cost_per_token(&self, model_id: &str) -> Option<ModelCostV2> {
match self.version {
1 => {
let input_cost = self.costs.iter().find(|cost| cost.matches(model_id, false));
let output_cost = self.costs.iter().find(|cost| cost.matches(model_id, true));

// V1 costs were defined per 1k tokens, so we need to convert to per token.
if input_cost.is_some() || output_cost.is_some() {
Some(ModelCostV2 {
input_per_token: input_cost.map_or(0.0, |c| c.cost_per_1k_tokens / 1000.0),
output_per_token: output_cost
.map_or(0.0, |c| c.cost_per_1k_tokens / 1000.0),
output_reasoning_per_token: 0.0, // in v1 this info is not available
input_cached_per_token: 0.0, // in v1 this info is not available
})
} else {
None
}
}
2 => self.models.get(model_id).copied(),
_ => None,
if self.is_enabled() {
self.models.get(model_id).copied()
} else {
None
}
}
}

/// A mapping of AI model types (like GPT-4) to their respective costs.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ModelCost {
pub(crate) model_id: LazyGlob,
pub(crate) for_completion: bool,
pub(crate) cost_per_1k_tokens: f64,
}

impl ModelCost {
/// `true` if this cost definition matches the given model.
pub fn matches(&self, model_id: &str, for_completion: bool) -> bool {
self.for_completion == for_completion && self.model_id.compiled().is_match(model_id)
}
}

/// Version 2 of a mapping of AI model types (like GPT-4) to their respective costs.
/// Version 1 had some limitations, so we're moving to a more flexible format.
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct ModelCostV2 {
Expand Down Expand Up @@ -366,26 +313,6 @@ mod tests {
/// Test that integer versions are handled correctly in the struct format
#[test]
fn test_model_cost_version_sent_as_number() {
// Test integer version 1
let original = r#"{"version":1,"costs":[{"modelId":"babbage-002.ft","forCompletion":false,"costPer1kTokens":0.0016}]}"#;
let deserialized: ModelCosts = serde_json::from_str(original).unwrap();
assert_debug_snapshot!(
deserialized,
@r#"
ModelCosts {
version: 1,
costs: [
ModelCost {
model_id: LazyGlob("babbage-002.ft"),
for_completion: false,
cost_per_1k_tokens: 0.0016,
},
],
models: {},
}
"#,
);

// Test integer version 2
let original_v2 = r#"{"version":2,"models":{"gpt-4":{"inputPerToken":0.03,"outputPerToken":0.06,"outputReasoningPerToken":0.12,"inputCachedPerToken":0.015}}}"#;
let deserialized_v2: ModelCosts = serde_json::from_str(original_v2).unwrap();
Expand All @@ -394,7 +321,6 @@ mod tests {
@r###"
ModelCosts {
version: 2,
costs: [],
models: {
"gpt-4": ModelCostV2 {
input_per_token: 0.03,
Expand All @@ -408,42 +334,19 @@ mod tests {
);

// Test unknown integer version
let original_unknown = r#"{"version":99,"costs":[]}"#;
let original_unknown = r#"{"version":99,"models":{}}"#;
let deserialized_unknown: ModelCosts = serde_json::from_str(original_unknown).unwrap();
assert_eq!(deserialized_unknown.version, 99);
assert!(!deserialized_unknown.is_enabled());
}

#[test]
fn test_model_cost_config_v1() {
let original = r#"{"version":1,"costs":[{"modelId":"babbage-002.ft","forCompletion":false,"costPer1kTokens":0.0016}]}"#;
let deserialized: ModelCosts = serde_json::from_str(original).unwrap();
assert_debug_snapshot!(deserialized, @r###"
ModelCosts {
version: 1,
costs: [
ModelCost {
model_id: LazyGlob("babbage-002.ft"),
for_completion: false,
cost_per_1k_tokens: 0.0016,
},
],
models: {},
}
"###);

let serialized = serde_json::to_string(&deserialized).unwrap();
assert_eq!(&serialized, original);
}

#[test]
fn test_model_cost_config_v2() {
let original = r#"{"version":2,"models":{"gpt-4":{"inputPerToken":0.03,"outputPerToken":0.06,"outputReasoningPerToken":0.12,"inputCachedPerToken":0.015}}}"#;
let deserialized: ModelCosts = serde_json::from_str(original).unwrap();
assert_debug_snapshot!(deserialized, @r###"
ModelCosts {
version: 2,
costs: [],
models: {
"gpt-4": ModelCostV2 {
input_per_token: 0.03,
Expand All @@ -459,48 +362,6 @@ mod tests {
assert_eq!(&serialized, original);
}

#[test]
fn test_model_cost_functionality_v1_only_input_tokens() {
// Test V1 functionality
let v1_config = ModelCosts {
version: 1,
costs: vec![ModelCost {
model_id: LazyGlob::new("gpt-4*"),
for_completion: false,
cost_per_1k_tokens: 0.03,
}],
models: HashMap::new(),
};
assert!(v1_config.is_enabled());
let costs = v1_config.cost_per_token("gpt-4-turbo").unwrap();
assert_eq!(costs.input_per_token * 1000.0, 0.03); // multiplying by 1000 to avoid floating point errors
assert_eq!(costs.output_per_token, 0.0); // output tokens are not defined
}

#[test]
fn test_model_cost_functionality_v1() {
let v1_config = ModelCosts {
version: 1,
costs: vec![
ModelCost {
model_id: LazyGlob::new("gpt-4*"),
for_completion: false,
cost_per_1k_tokens: 0.03,
},
ModelCost {
model_id: LazyGlob::new("gpt-4*"),
for_completion: true,
cost_per_1k_tokens: 0.06,
},
],
models: HashMap::new(),
};
assert!(v1_config.is_enabled());
let costs = v1_config.cost_per_token("gpt-4").unwrap();
assert_eq!(costs.input_per_token * 1000.0, 0.03); // multiplying by 1000 to avoid floating point errors
assert_eq!(costs.output_per_token * 1000.0, 0.06); // multiplying by 1000 to avoid floating point errors
}

#[test]
fn test_model_cost_functionality_v2() {
// Test V2 functionality
Expand All @@ -516,7 +377,6 @@ mod tests {
);
let v2_config = ModelCosts {
version: 2,
costs: vec![],
models: models_map,
};
assert!(v2_config.is_enabled());
Expand Down
Loading