Skip to content
6 changes: 5 additions & 1 deletion crates/openfang-api/src/openai_compat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,11 @@ fn convert_messages(oai_messages: &[OaiMessage]) -> Vec<Message> {
.unwrap_or(parts[0])
.to_string();
let data = parts[1].to_string();
Some(ContentBlock::Image { media_type, data })
Some(ContentBlock::Image {
media_type,
data,
source_url: None,
})
} else {
None
}
Expand Down
2 changes: 2 additions & 0 deletions crates/openfang-api/src/routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ pub fn resolve_attachments(
blocks.push(openfang_types::message::ContentBlock::Image {
media_type: content_type,
data: b64,
source_url: None,
});
}
Err(e) => {
Expand Down Expand Up @@ -512,6 +513,7 @@ pub async fn get_agent_session(
openfang_types::message::ContentBlock::Image {
media_type,
data,
..
} => {
texts.push("[Image]".to_string());
// Persist image to upload dir so it can be
Expand Down
72 changes: 71 additions & 1 deletion crates/openfang-channels/src/bridge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1453,7 +1453,14 @@ async fn download_image_to_blocks(url: &str, caption: Option<&str>) -> Vec<Conte
}
}

blocks.push(ContentBlock::Image { media_type, data });
blocks.push(ContentBlock::Image {
media_type,
data,
// Preserve the original CDN/source URL so text-only drivers (e.g.
// Claude Code) can reference it, and vision-capable drivers retain
// it for diagnostics.
source_url: Some(url.to_string()),
});

blocks
}
Expand Down Expand Up @@ -2233,6 +2240,7 @@ mod tests {
ContentBlock::Image {
media_type: "image/jpeg".to_string(),
data: "base64data".to_string(),
source_url: None,
},
];

Expand All @@ -2255,6 +2263,7 @@ mod tests {
let blocks = vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: "base64data".to_string(),
source_url: None,
}];

// Default impl sends empty text when no text blocks
Expand Down Expand Up @@ -2458,4 +2467,65 @@ mod tests {
"image/jpeg"
);
}

/// Regression test: `download_image_to_blocks` must populate the
/// `source_url` field on the resulting `ContentBlock::Image`. Text-only
/// drivers (e.g. Claude Code) rely on this URL to reference the image
/// without re-uploading bytes; a regression here silently breaks vision
/// for those drivers.
///
/// Spins up a local TCP listener that serves a stub PNG response so we
/// can drive the function end-to-end without external dependencies.
#[tokio::test]
async fn download_image_to_blocks_populates_source_url() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;

let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let url = format!("http://{addr}/test.png");

// Body is arbitrary — the function trusts the Content-Type header
// when it starts with `image/`. PNG signature is included so any
// future magic-byte fallback also matches.
let body: &[u8] = b"\x89PNG\r\n\x1a\nfakepngbytes";
let response_head = format!(
"HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);

tokio::spawn(async move {
let (mut sock, _) = listener.accept().await.unwrap();
// Drain the request — we don't care what reqwest sent.
let mut buf = [0u8; 1024];
let _ = sock.read(&mut buf).await;
sock.write_all(response_head.as_bytes()).await.unwrap();
sock.write_all(body).await.unwrap();
let _ = sock.shutdown().await;
});

let blocks = download_image_to_blocks(&url, Some("hello")).await;

// Caption first, image second.
assert_eq!(blocks.len(), 2, "expected caption + image blocks");
match &blocks[0] {
ContentBlock::Text { text, .. } => assert_eq!(text, "hello"),
other => panic!("expected text caption block, got {other:?}"),
}
match &blocks[1] {
ContentBlock::Image {
source_url,
media_type,
..
} => {
assert_eq!(
source_url.as_deref(),
Some(url.as_str()),
"source_url must round-trip the fetched URL"
);
assert_eq!(media_type, "image/png");
}
other => panic!("expected image block, got {other:?}"),
}
}
}
1 change: 1 addition & 0 deletions crates/openfang-runtime/src/agent_loop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3327,6 +3327,7 @@ mod tests {
ContentBlock::Image {
media_type: "image/png".to_string(),
data: "aGVsbG8=".to_string(),
source_url: None,
}
}

Expand Down
97 changes: 95 additions & 2 deletions crates/openfang-runtime/src/compactor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -400,8 +400,31 @@ fn build_conversation_text(messages: &[Message], config: &CompactionConfig) -> S
conversation_text
.push_str(&format!("[Tool result ({status}): {preview}]\n\n"));
}
ContentBlock::Image { media_type, .. } => {
conversation_text.push_str(&format!("[Image: {media_type}]\n\n"));
ContentBlock::Image {
media_type,
source_url,
..
} => {
// Preserve the original CDN URL across compaction so the
// outbound Discord path (PR-C) can re-attach the image by
// re-fetching it. Only http(s) URLs are exposed: local
// `file://` tmpfile paths are an internal materialization
// detail and shouldn't leak into compacted summaries that
// may be persisted, logged, or sent across processes.
match source_url.as_deref() {
Some(url)
if url.starts_with("http://")
|| url.starts_with("https://") =>
{
conversation_text.push_str(&format!(
"[Image: {media_type} @ {url}]\n\n"
));
}
_ => {
conversation_text
.push_str(&format!("[Image: {media_type}]\n\n"));
}
}
}
ContentBlock::Thinking { .. } => {}
ContentBlock::Unknown => {}
Expand Down Expand Up @@ -1266,6 +1289,7 @@ mod tests {
content: MessageContent::Blocks(vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: "base64data".to_string(),
source_url: None,
}]),
},
];
Expand All @@ -1278,6 +1302,75 @@ mod tests {
assert!(text.contains("[Image: image/png]"));
}

#[test]
fn test_build_conversation_text_image_source_url_https() {
// https:// CDN URL is exposed post-compaction so the outbound path
// can re-fetch the image.
let config = CompactionConfig::default();
let messages = vec![Message {
role: Role::User,
content: MessageContent::Blocks(vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: "base64data".to_string(),
source_url: Some("https://cdn.discordapp.com/attachments/x/y.png".to_string()),
}]),
}];
let text = build_conversation_text(&messages, &config);
assert!(
text.contains("[Image: image/png @ https://cdn.discordapp.com/attachments/x/y.png]"),
"https source_url should be preserved, got: {text}"
);
}

#[test]
fn test_build_conversation_text_image_source_url_http() {
// Plain http (rare but valid) is also exposed.
let config = CompactionConfig::default();
let messages = vec![Message {
role: Role::User,
content: MessageContent::Blocks(vec![ContentBlock::Image {
media_type: "image/jpeg".to_string(),
data: "base64data".to_string(),
source_url: Some("http://example.com/foo.jpg".to_string()),
}]),
}];
let text = build_conversation_text(&messages, &config);
assert!(
text.contains("[Image: image/jpeg @ http://example.com/foo.jpg]"),
"http source_url should be preserved, got: {text}"
);
}

#[test]
fn test_build_conversation_text_image_source_url_file_falls_back() {
// file:// URLs (local tmpfile materialization) MUST NOT leak into
// compacted summaries — fall back to the legacy mime-only form.
let config = CompactionConfig::default();
let messages = vec![Message {
role: Role::User,
content: MessageContent::Blocks(vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: "base64data".to_string(),
source_url: Some(
"file:///Users/x/.openfang/tmp/images/abc.png".to_string(),
),
}]),
}];
let text = build_conversation_text(&messages, &config);
assert!(
text.contains("[Image: image/png]"),
"file:// source_url should fall back to legacy form, got: {text}"
);
assert!(
!text.contains("file://"),
"file:// path must not leak post-compaction, got: {text}"
);
assert!(
!text.contains(".openfang"),
"local tmpfile path must not leak post-compaction, got: {text}"
);
}

#[test]
fn test_build_conversation_text_truncates_oversized() {
let config = CompactionConfig {
Expand Down
2 changes: 1 addition & 1 deletion crates/openfang-runtime/src/drivers/anthropic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -664,7 +664,7 @@ fn convert_message(msg: &Message) -> ApiMessage {
ContentBlock::Text { text, .. } => {
Some(ApiContentBlock::Text { text: text.clone() })
}
ContentBlock::Image { media_type, data } => Some(ApiContentBlock::Image {
ContentBlock::Image { media_type, data, .. } => Some(ApiContentBlock::Image {
source: ApiImageSource {
source_type: "base64".to_string(),
media_type: media_type.clone(),
Expand Down
Loading
Loading