Owlibou/owlen
2
1
Fork 0
Code Issues 20 Pull Requests Actions Packages Projects Releases 5 Activity

feat: enable multimodal attachments for agents

Browse Source
This commit is contained in:
vikingowl
2025-10-26 05:14:17 +01:00
parent 353c0a8239
commit 28b6eb0a9a
20 changed files with 1387 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crates/owlen-cli/Cargo.toml
View File

@@ -54,6 +54,9 @@ serde_json = { workspace = true }
regex = { workspace = true }
thiserror = { workspace = true }
dirs = { workspace = true }
base64 = { workspace = true }
mime_guess = { workspace = true }
image = { workspace = true }
[dev-dependencies]
tokio = { workspace = true }

236
crates/owlen-cli/src/agent_main.rs
View File

@@ -6,11 +6,25 @@
//! OllamaProvider, a RemoteMcpClient, runs the AgentExecutor and prints the
//! final answer.
use std::sync::Arc;
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use clap::Parser;
use anyhow::Context;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use clap::{Parser, builder::ValueHint};
use image::{self, GenericImageView, imageops::FilterType};
use owlen_cli::agent::{AgentConfig, AgentExecutor};
use owlen_core::mcp::remote_client::RemoteMcpClient;
use owlen_core::{mcp::remote_client::RemoteMcpClient, types::MessageAttachment};
use tokio::fs;
const MAX_ATTACHMENT_BYTES: u64 = 8 * 1024 * 1024;
const ATTACHMENT_ASCII_WIDTH: u32 = 24;
const ATTACHMENT_ASCII_HEIGHT: u32 = 12;
const ATTACHMENT_TEXT_PREVIEW_LINES: usize = 12;
const ATTACHMENT_TEXT_PREVIEW_WIDTH: usize = 80;
const ATTACHMENT_INLINE_PREVIEW_LINES: usize = 6;
/// Command‑line arguments for the agent binary.
#[derive(Parser, Debug)]
@@ -23,6 +37,9 @@ use owlen_core::mcp::remote_client::RemoteMcpClient;
struct Args {
/// The initial user query.
prompt: String,
/// Paths to files that should be sent with the initial turn.
#[arg(long = "attach", short = 'a', value_name = "PATH", value_hint = ValueHint::FilePath)]
attachments: Vec<PathBuf>,
/// Model to use (defaults to Ollama default).
#[arg(long)]
model: Option<String>,
@@ -34,6 +51,26 @@ struct Args {
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let args = Args::parse();
let Args {
prompt,
attachments: attachment_paths,
model,
max_iter,
} = args;
let attachments = load_attachments(&attachment_paths).await?;
if !attachments.is_empty() {
println!(
"Attaching {} {}:",
attachments.len(),
if attachments.len() == 1 {
"artifact"
} else {
"artifacts"
}
);
render_attachment_previews(&attachments);
}
// Initialise the MCP LLM client – it implements Provider and talks to the
// MCP LLM server which wraps Ollama. This ensures all communication goes
@@ -44,13 +81,13 @@ async fn main() -> anyhow::Result<()> {
let mcp_client = Arc::clone(&provider) as Arc<RemoteMcpClient>;
let config = AgentConfig {
max_iterations: args.max_iter,
model: args.model.unwrap_or_else(|| "llama3.2:latest".to_string()),
max_iterations: max_iter,
model: model.unwrap_or_else(|| "llama3.2:latest".to_string()),
..AgentConfig::default()
};
let executor = AgentExecutor::new(provider, mcp_client, config);
match executor.run(args.prompt).await {
match executor.run_with_attachments(prompt, attachments).await {
Ok(result) => {
println!("\n✓ Agent completed in {} iterations", result.iterations);
println!("\nFinal answer:\n{}", result.answer);
@@ -59,3 +96,190 @@ async fn main() -> anyhow::Result<()> {
Err(e) => Err(anyhow::anyhow!(e)),
}
}
async fn load_attachments(paths: &[PathBuf]) -> anyhow::Result<Vec<MessageAttachment>> {
let mut attachments = Vec::new();
for path in paths {
let attachment = load_attachment(path).await?;
attachments.push(attachment);
}
Ok(attachments)
}
async fn load_attachment(path: &Path) -> anyhow::Result<MessageAttachment> {
let metadata = fs::metadata(path)
.await
.with_context(|| format!("Unable to inspect {}", path.display()))?;
if !metadata.is_file() {
return Err(anyhow::anyhow!("{} is not a regular file", path.display()));
}
if metadata.len() > MAX_ATTACHMENT_BYTES {
return Err(anyhow::anyhow!(
"Attachments are limited to {} (requested {}): {}",
format_attachment_size(MAX_ATTACHMENT_BYTES),
format_attachment_size(metadata.len()),
path.display()
));
}
let bytes = fs::read(path)
.await
.with_context(|| format!("Failed to read {}", path.display()))?;
let mime = mime_guess::from_path(path).first_or_octet_stream();
let mime_string = mime.essence_str().to_string();
let file_name = path
.file_name()
.and_then(|value| value.to_str())
.unwrap_or("attachment")
.to_string();
let is_text = mime_string.starts_with("text/") || std::str::from_utf8(&bytes).is_ok();
let mut preview_lines = Vec::new();
let mut attachment = if is_text {
let text = String::from_utf8_lossy(&bytes).into_owned();
preview_lines = preview_lines_for_text(&text);
let mut attachment =
MessageAttachment::from_text(Some(file_name.clone()), mime_string.clone(), text);
attachment.size_bytes = Some(metadata.len());
attachment
} else {
if mime_string.starts_with("image/")
&& let Some(lines) = preview_lines_for_image(&bytes)
{
preview_lines = lines;
}
let encoded = BASE64_STANDARD.encode(&bytes);
MessageAttachment::from_base64(
file_name.clone(),
mime_string.clone(),
encoded,
Some(metadata.len()),
)
};
attachment.size_bytes = Some(metadata.len());
attachment = attachment.with_source_path(path.to_path_buf());
if !preview_lines.is_empty() {
attachment = attachment.with_preview_lines(preview_lines);
}
Ok(attachment)
}
fn render_attachment_previews(attachments: &[MessageAttachment]) {
for (idx, attachment) in attachments.iter().enumerate() {
println!(" {}. {}", idx + 1, summarize_attachment(attachment));
if let Some(lines) = attachment.preview_lines.as_ref() {
for line in lines.iter().take(ATTACHMENT_INLINE_PREVIEW_LINES) {
println!(" {}", line);
}
if lines.len() > ATTACHMENT_INLINE_PREVIEW_LINES {
println!(" …");
}
}
}
if !attachments.is_empty() {
println!();
}
}
fn summarize_attachment(attachment: &MessageAttachment) -> String {
let icon = if attachment.is_image() {
"📷"
} else if attachment
.mime_type
.to_ascii_lowercase()
.starts_with("text/")
{
"📄"
} else {
"📎"
};
let name = attachment
.name
.as_deref()
.unwrap_or(attachment.mime_type.as_str());
let mut parts = vec![format!("{icon} {name}"), attachment.mime_type.clone()];
if let Some(size) = attachment.size_bytes {
parts.push(format_attachment_size(size));
}
parts.join(" · ")
}
fn format_attachment_size(bytes: u64) -> String {
const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"];
let mut value = bytes as f64;
let mut index = 0usize;
while value >= 1024.0 && index < UNITS.len() - 1 {
value /= 1024.0;
index += 1;
}
if index == 0 {
format!("{bytes} {}", UNITS[index])
} else {
format!("{value:.1} {}", UNITS[index])
}
}
fn preview_lines_for_text(text: &str) -> Vec<String> {
let mut lines = Vec::new();
for raw in text.lines().take(ATTACHMENT_TEXT_PREVIEW_LINES) {
let trimmed = raw.trim_end();
if trimmed.is_empty() {
lines.push(String::new());
continue;
}
let mut snippet = trimmed
.chars()
.take(ATTACHMENT_TEXT_PREVIEW_WIDTH)
.collect::<String>();
if trimmed.chars().count() > ATTACHMENT_TEXT_PREVIEW_WIDTH {
snippet.push('…');
}
lines.push(snippet);
}
if lines.is_empty() {
lines.push("(empty attachment)".to_string());
}
lines
}
fn preview_lines_for_image(bytes: &[u8]) -> Option<Vec<String>> {
let image = image::load_from_memory(bytes).ok()?;
let (width, height) = image.dimensions();
let mut lines = Vec::new();
lines.push(format!("{width} × {height} px"));
let target_width = ATTACHMENT_ASCII_WIDTH;
let target_height = ATTACHMENT_ASCII_HEIGHT;
let scale = (target_width as f32 / width as f32)
.min(target_height as f32 / height as f32)
.clamp(0.05, 1.0);
let scaled_width = (width as f32 * scale).max(1.0).round() as u32;
let scaled_height = (height as f32 * scale).max(1.0).round() as u32;
let resized = image
.resize_exact(
scaled_width.max(1),
scaled_height.max(1),
FilterType::Triangle,
)
.to_luma8();
const PALETTE: [char; 10] = [' ', '.', ':', '-', '=', '+', '*', '#', '%', '@'];
for y in 0..resized.height() {
let mut row = String::with_capacity((resized.width() as usize) * 2);
for x in 0..resized.width() {
let luminance = resized.get_pixel(x, y)[0] as usize;
let idx = luminance * (PALETTE.len() - 1) / 255;
let ch = PALETTE[idx];
row.push(ch);
row.push(ch);
}
lines.push(row);
}
Some(lines)
}

83
crates/owlen-cli/src/commands/cloud.rs
View File

@@ -25,7 +25,7 @@ const CLOUD_PROVIDER_KEY: &str = "ollama_cloud";
pub enum CloudCommand {
/// Configure Ollama Cloud credentials
Setup {
/// API key passed directly on the command line (prompted when omitted)
/// API key passed directly on the command line
#[arg(long)]
api_key: Option<String>,
/// Override the cloud endpoint (default: https://ollama.com)
@@ -89,17 +89,39 @@ async fn setup(
configure_cloud_endpoint(entry, &endpoint, force_cloud_base_url)
};
let key = match api_key {
Some(value) if !value.trim().is_empty() => value,
_ => {
let prompt = format!("Enter API key for {provider}: ");
encryption::prompt_password(&prompt)?
}
};
let mut credential_manager: Option<Arc<CredentialManager>> = None;
if config.privacy.encrypt_local_data {
let storage = Arc::new(StorageManager::new().await?);
let manager = unlock_credential_manager(&config, storage.clone())?;
credential_manager = Some(unlock_credential_manager(&config, storage)?);
}
let mut key_opt = api_key.filter(|value| !value.trim().is_empty());
if key_opt.is_none() {
if let Some(manager) = credential_manager.as_ref() {
if let Some(credentials) = manager.get_credentials(OLLAMA_CLOUD_CREDENTIAL_ID).await? {
key_opt = Some(credentials.api_key);
}
} else if let Some(existing) = config
.provider(&provider)
.and_then(|cfg| cfg.api_key.clone())
{
key_opt = Some(existing);
}
}
let key = key_opt
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
.ok_or_else(|| {
anyhow!(
"API key is required when configuring provider `{provider}`. \
Supply the --api-key flag, set providers.{provider}.api_key in config.toml, \
or populate the credential vault."
)
})?;
if let Some(manager) = credential_manager.clone() {
let credentials = ApiCredentials {
api_key: key.clone(),
endpoint: endpoint.clone(),
@@ -385,46 +407,7 @@ fn vault_path(storage: &StorageManager) -> Result<PathBuf> {
}
fn unlock_vault(path: &Path) -> Result<encryption::VaultHandle> {
use std::env;
if path.exists() {
if let Some(password) = env::var("OWLEN_MASTER_PASSWORD")
.ok()
.map(|value| value.trim().to_string())
.filter(|password| !password.is_empty())
{
return encryption::unlock_with_password(path.to_path_buf(), &password)
.context("Failed to unlock vault with OWLEN_MASTER_PASSWORD");
}
for attempt in 0..3 {
let password = encryption::prompt_password("Enter master password: ")?;
match encryption::unlock_with_password(path.to_path_buf(), &password) {
Ok(handle) => {
set_env_var("OWLEN_MASTER_PASSWORD", password);
return Ok(handle);
}
Err(err) => {
eprintln!("Failed to unlock vault: {err}");
if attempt == 2 {
return Err(err);
}
}
}
}
bail!("Unable to unlock encrypted credential vault");
}
let handle = encryption::unlock_interactive(path.to_path_buf())?;
if env::var("OWLEN_MASTER_PASSWORD")
.map(|v| v.trim().is_empty())
.unwrap_or(true)
{
let password = encryption::prompt_password("Cache master password for this session: ")?;
set_env_var("OWLEN_MASTER_PASSWORD", password);
}
Ok(handle)
encryption::unlock(path.to_path_buf())
}
async fn hydrate_api_key(

2
crates/owlen-core/Cargo.toml
View File

@@ -37,7 +37,6 @@ keyring = { workspace = true }
chrono = { workspace = true }
crossterm = { workspace = true }
urlencoding = { workspace = true }
rpassword = { workspace = true }
sqlx = { workspace = true }
reqwest = { workspace = true, features = ["default"] }
path-clean = "1.0"
@@ -46,6 +45,7 @@ tokio-tungstenite = "0.21"
tungstenite = "0.21"
ollama-rs = { version = "0.3", features = ["stream", "headers"] }
once_cell = { workspace = true }
base64 = { workspace = true }
[dev-dependencies]
tokio-test = { workspace = true }

13
crates/owlen-core/src/agent.rs
View File

@@ -5,7 +5,7 @@
use crate::Provider;
use crate::mcp::{McpClient, McpToolCall, McpToolDescriptor, McpToolResponse};
use crate::types::{ChatParameters, ChatRequest, Message};
use crate::types::{ChatParameters, ChatRequest, Message, MessageAttachment};
use crate::{Error, Result, SubAgentSpec};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
@@ -140,7 +140,16 @@ impl AgentExecutor {
/// Run the agent loop with the given query
pub async fn run(&self, query: String) -> Result<AgentResult> {
let mut messages = vec![Message::user(query)];
self.run_with_attachments(query, Vec::new()).await
}
/// Run the agent loop with an initial multimodal payload.
pub async fn run_with_attachments(
&self,
query: String,
attachments: Vec<MessageAttachment>,
) -> Result<AgentResult> {
let mut messages = vec![Message::user(query).with_attachments(attachments)];
let tools = self.discover_tools().await?;
for iteration in 0..self.config.max_iterations {

12
crates/owlen-core/src/conversation.rs
View File

@@ -1,6 +1,6 @@
use crate::Result;
use crate::storage::StorageManager;
use crate::types::{Conversation, Message};
use crate::types::{Conversation, Message, MessageAttachment};
use serde_json::{Number, Value};
use std::collections::{HashMap, VecDeque};
use std::time::{Duration, Instant};
@@ -104,6 +104,16 @@ impl ConversationManager {
self.register_message(message)
}
/// Add a user message that includes rich attachments.
pub fn push_user_message_with_attachments(
&mut self,
content: impl Into<String>,
attachments: Vec<MessageAttachment>,
) -> Uuid {
let message = Message::user(content.into()).with_attachments(attachments);
self.register_message(message)
}
/// Add a system message and return its identifier
pub fn push_system_message(&mut self, content: impl Into<String>) -> Uuid {
let message = Message::system(content.into());

139
crates/owlen-core/src/encryption.rs
View File

@@ -1,13 +1,13 @@
use std::collections::HashMap;
use std::fs;
use std::path::PathBuf;
use std::fs::{self, OpenOptions};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use aes_gcm::{
Aes256Gcm, Nonce,
aead::{Aead, KeyInit},
};
use anyhow::{Context, Result, bail};
use ring::digest;
use ring::rand::{SecureRandom, SystemRandom};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
@@ -54,9 +54,14 @@ impl VaultHandle {
}
impl EncryptedStorage {
pub fn new(storage_path: PathBuf, password: &str) -> Result<Self> {
let digest = digest::digest(&digest::SHA256, password.as_bytes());
let cipher = Aes256Gcm::new_from_slice(digest.as_ref())
pub fn new(storage_path: PathBuf, key: &[u8]) -> Result<Self> {
if key.len() != 32 {
bail!(
"Invalid key length for encrypted storage ({}). Expected 32 bytes for AES-256.",
key.len()
);
}
let cipher = Aes256Gcm::new_from_slice(key)
.map_err(|_| anyhow::anyhow!("Invalid key length for AES-256"))?;
if let Some(parent) = storage_path.parent() {
@@ -141,63 +146,13 @@ impl EncryptedStorage {
}
}
pub fn prompt_password(prompt: &str) -> Result<String> {
let password = rpassword::prompt_password(prompt)
.map_err(|e| anyhow::anyhow!("Failed to read password: {e}"))?;
if password.is_empty() {
bail!("Password cannot be empty");
}
Ok(password)
}
pub fn prompt_new_password() -> Result<String> {
loop {
let first = prompt_password("Enter new master password: ")?;
let confirm = prompt_password("Confirm master password: ")?;
if first == confirm {
return Ok(first);
}
println!("Passwords did not match. Please try again.");
}
}
pub fn unlock_with_password(storage_path: PathBuf, password: &str) -> Result<VaultHandle> {
let storage = EncryptedStorage::new(storage_path, password)?;
pub fn unlock(storage_path: PathBuf) -> Result<VaultHandle> {
let key = load_or_create_encryption_key(&storage_path)?;
let storage = EncryptedStorage::new(storage_path, &key)?;
let data = load_or_initialize_vault(&storage)?;
Ok(VaultHandle { storage, data })
}
pub fn unlock_interactive(storage_path: PathBuf) -> Result<VaultHandle> {
if storage_path.exists() {
for attempt in 0..3 {
let password = prompt_password("Enter master password: ")?;
match unlock_with_password(storage_path.clone(), &password) {
Ok(handle) => return Ok(handle),
Err(err) => {
println!("Failed to unlock vault: {err}");
if attempt == 2 {
return Err(err);
}
}
}
}
bail!("Failed to unlock encrypted storage after multiple attempts");
} else {
println!(
"No encrypted storage found at {}. Initializing a new vault.",
storage_path.display()
);
let password = prompt_new_password()?;
let storage = EncryptedStorage::new(storage_path, &password)?;
let data = VaultData {
master_key: generate_master_key()?,
..Default::default()
};
storage.store(&data)?;
Ok(VaultHandle { storage, data })
}
}
fn load_or_initialize_vault(storage: &EncryptedStorage) -> Result<VaultData> {
match storage.load::<VaultData>() {
Ok(data) => {
@@ -224,6 +179,72 @@ fn load_or_initialize_vault(storage: &EncryptedStorage) -> Result<VaultData> {
}
}
fn key_path(storage_path: &Path) -> PathBuf {
let mut path = storage_path.to_path_buf();
path.set_extension("key");
path
}
fn load_or_create_encryption_key(storage_path: &Path) -> Result<Vec<u8>> {
let key_path = key_path(storage_path);
match fs::read(&key_path) {
Ok(bytes) => {
if bytes.len() == 32 {
Ok(bytes)
} else {
bail!(
"Invalid encryption key length stored in {} ({} bytes). Expected 32 bytes.",
key_path.display(),
bytes.len()
);
}
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {
let key = generate_master_key()?;
write_key_file(&key_path, &key)?;
Ok(key)
}
Err(err) => Err(err)
.with_context(|| format!("Failed to read encryption key from {}", key_path.display())),
}
}
fn write_key_file(path: &Path, key: &[u8]) -> Result<()> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create directory {}", parent.display()))?;
}
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
let mut file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.mode(0o600)
.open(path)
.with_context(|| format!("Failed to open encryption key file {}", path.display()))?;
file.write_all(key)
.with_context(|| format!("Failed to write encryption key file {}", path.display()))?;
}
#[cfg(not(unix))]
{
let mut file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(path)
.with_context(|| format!("Failed to open encryption key file {}", path.display()))?;
file.write_all(key)
.with_context(|| format!("Failed to write encryption key file {}", path.display()))?;
}
Ok(())
}
fn generate_master_key() -> Result<Vec<u8>> {
let mut key = vec![0u8; 32];
SystemRandom::new()

77
crates/owlen-core/src/providers/ollama.rs
View File

@@ -2,7 +2,7 @@
use std::{
collections::{HashMap, HashSet},
convert::TryFrom,
env,
env, fs,
net::{SocketAddr, TcpStream},
pin::Pin,
process::Command,
@@ -11,19 +11,23 @@ use std::{
};
use anyhow::anyhow;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use futures::{Stream, StreamExt, future::BoxFuture, future::join_all};
use log::{debug, warn};
use ollama_rs::{
Ollama,
error::OllamaError,
generation::chat::{
ChatMessage as OllamaMessage, ChatMessageResponse as OllamaChatResponse,
MessageRole as OllamaRole, request::ChatMessageRequest as OllamaChatRequest,
},
generation::tools::{
ToolCall as OllamaToolCall, ToolCallFunction as OllamaToolCallFunction,
ToolInfo as OllamaToolInfo,
},
generation::{
chat::{
ChatMessage as OllamaMessage, ChatMessageResponse as OllamaChatResponse,
MessageRole as OllamaRole, request::ChatMessageRequest as OllamaChatRequest,
},
images::Image,
},
headers::{AUTHORIZATION, HeaderMap, HeaderValue},
models::{LocalModel, ModelInfo as OllamaModelInfo, ModelOptions},
};
@@ -50,7 +54,8 @@ use crate::{
model::{DetailedModelInfo, ModelDetailsCache, ModelManager},
provider::{ProviderError, ProviderErrorKind},
types::{
ChatParameters, ChatRequest, ChatResponse, Message, ModelInfo, Role, TokenUsage, ToolCall,
ChatParameters, ChatRequest, ChatResponse, Message, MessageAttachment, ModelInfo, Role,
TokenUsage, ToolCall,
},
};
@@ -1665,6 +1670,7 @@ fn convert_message(message: Message) -> OllamaMessage {
content,
metadata,
tool_calls,
attachments,
..
} = message;
@@ -1690,11 +1696,43 @@ fn convert_message(message: Message) -> OllamaMessage {
.get("thinking")
.and_then(|value| value.as_str().map(|s| s.to_owned()));
let images: Vec<Image> = attachments
.into_iter()
.filter_map(|attachment| {
if !attachment.is_image() {
return None;
}
if let Some(data) = attachment.data_base64 {
return Some(Image::from_base64(data));
}
if let Some(path) = attachment.source_path {
match fs::read(&path) {
Ok(bytes) => {
let encoded = BASE64_STANDARD.encode(bytes);
return Some(Image::from_base64(encoded));
}
Err(err) => {
warn!(
"Failed to read attachment '{}' for image conversion: {}",
path.display(),
err
);
}
}
}
None
})
.collect();
OllamaMessage {
role,
content,
tool_calls,
images: None,
images: if images.is_empty() {
None
} else {
Some(images)
},
thinking,
}
}
@@ -1729,6 +1767,30 @@ fn convert_ollama_message(message: OllamaMessage) -> Message {
metadata.insert("thinking".to_string(), Value::String(thinking));
}
let attachments = message
.images
.unwrap_or_default()
.into_iter()
.enumerate()
.filter_map(|(idx, image)| {
let data = image.to_base64();
if data.is_empty() {
return None;
}
let size_bytes = (data.len() as u64).saturating_mul(3).saturating_div(4);
let name = format!("image-{}.png", idx + 1);
Some(
MessageAttachment::from_base64(
name,
"image/png",
data.to_string(),
Some(size_bytes),
)
.with_description(format!("Generated image {}", idx + 1)),
)
})
.collect();
Message {
id: Uuid::new_v4(),
role,
@@ -1736,6 +1798,7 @@ fn convert_ollama_message(message: OllamaMessage) -> Message {
metadata,
timestamp: SystemTime::now(),
tool_calls,
attachments,
}
}

94
crates/owlen-core/src/session.rs
View File

@@ -65,11 +65,29 @@ fn estimate_tokens(messages: &[Message]) -> u32 {
fn estimate_message_tokens(message: &Message) -> u32 {
let content = message.content.trim();
if content.is_empty() {
return 4;
}
let approx = max(4, content.chars().count() / 4 + 1);
(approx + 4) as u32
let base = if content.is_empty() {
4
} else {
let approx = max(4, content.chars().count() / 4 + 1);
approx + 4
} as u32;
message.attachments.iter().fold(base, |acc, attachment| {
let mut bonus: u32 = if attachment.is_image() { 256 } else { 64 };
if let Some(text) = attachment.text_data() {
let text_tokens = max(4, text.chars().count() / 4 + 1) as u32;
bonus = bonus.max(text_tokens);
}
if let Some(size) = attachment.size_bytes {
let kilobytes = ((size as f64) / 1024.0).ceil() as u32;
let size_tokens = kilobytes.saturating_mul(8).max(32);
bonus = bonus.max(size_tokens);
}
acc.saturating_add(bonus)
})
}
fn build_transcript(messages: &[Message]) -> String {
@@ -84,9 +102,37 @@ fn build_transcript(messages: &[Message]) -> String {
};
let snippet = sanitize_snippet(&message.content);
if snippet.is_empty() {
continue;
if message.attachments.is_empty() {
continue;
}
}
transcript.push_str(&format!("{role}: {snippet}\n\n"));
transcript.push_str(&format!("{role}: {snippet}\n"));
if !message.attachments.is_empty() {
for attachment in &message.attachments {
let name = attachment
.name
.as_deref()
.unwrap_or(attachment.mime_type.as_str());
let mut summary = format!(" • Attachment {name} ({})", attachment.mime_type);
if let Some(size) = attachment.size_bytes {
let kb = (size as f64) / 1024.0;
summary.push_str(&format!(" · {:.1} KB", kb));
}
if let Some(text) = attachment.text_data() {
let trimmed = text.trim();
if !trimmed.is_empty() {
let mut preview = trimmed.chars().take(60).collect::<String>();
if trimmed.chars().count() > 60 {
preview.push('…');
}
summary.push_str(&format!(" · {}", preview));
}
}
transcript.push_str(&summary);
transcript.push('\n');
}
}
transcript.push('\n');
}
if messages.len() > take {
transcript.push_str(&format!(
@@ -137,6 +183,29 @@ fn collect_recent_by_role(messages: &[Message], role: Role, limit: usize) -> Vec
if results.len() == limit {
break;
}
continue;
}
if !message.attachments.is_empty() {
let names = message
.attachments
.iter()
.map(|attachment| {
attachment
.name
.as_deref()
.unwrap_or(attachment.mime_type.as_str())
})
.collect::<Vec<_>>()
.join(", ");
if !names.is_empty() {
results.push(format!("Attachments: {}", names));
} else {
results.push("Attachments received".to_string());
}
if results.len() == limit {
break;
}
}
}
}
@@ -792,12 +861,7 @@ impl SessionController {
.or_else(dirs::data_local_dir)
.unwrap_or_else(|| PathBuf::from("."));
let secure_path = base_dir.join("encrypted_data.json");
let handle = match env::var("OWLEN_MASTER_PASSWORD") {
Ok(password) if !password.is_empty() => {
encryption::unlock_with_password(secure_path, &password)?
}
_ => encryption::unlock_interactive(secure_path)?,
};
let handle = encryption::unlock(secure_path)?;
let master = Arc::new(handle.data.master_key.clone());
master_key = Some(master.clone());
vault_handle = Some(Arc::new(Mutex::new(handle)));
@@ -2432,10 +2496,6 @@ mod tests {
}
async fn build_session(server: &MockServer) -> (SessionController, tempfile::TempDir) {
unsafe {
std::env::set_var("OWLEN_MASTER_PASSWORD", "test-password");
}
let temp_dir = tempdir().expect("tempdir");
let storage_path = temp_dir.path().join("owlen.db");
let storage = Arc::new(

125
crates/owlen-core/src/types.rs
View File

@@ -3,6 +3,7 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt;
use std::path::PathBuf;
use uuid::Uuid;
/// A message in a conversation
@@ -21,6 +22,9 @@ pub struct Message {
/// Tool calls requested by the assistant
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_calls: Option<Vec<ToolCall>>,
/// Rich attachments (images, artifacts, files) associated with the message
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub attachments: Vec<MessageAttachment>,
}
/// Role of a message sender
@@ -48,6 +52,115 @@ pub struct ToolCall {
pub arguments: serde_json::Value,
}
fn default_mime_type() -> String {
"application/octet-stream".to_string()
}
/// Attachment associated with a message (image, artifact, or rich output).
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct MessageAttachment {
/// Unique identifier for this attachment instance.
pub id: Uuid,
/// Human friendly name, typically a filename.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
/// Optional descriptive text supplied by the sender.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
/// MIME type describing the payload.
#[serde(default = "default_mime_type")]
pub mime_type: String,
/// Source filesystem path if the attachment originated from disk.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source_path: Option<PathBuf>,
/// Binary payload encoded as base64, when applicable.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data_base64: Option<String>,
/// Inline UTF-8 payload when the attachment is textual.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_content: Option<String>,
/// Approximate size in bytes for UI hints.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub size_bytes: Option<u64>,
/// Optional pre-rendered preview lines for fast UI rendering.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub preview_lines: Option<Vec<String>>,
}
impl MessageAttachment {
/// Build an attachment from base64 encoded binary data.
pub fn from_base64(
name: impl Into<String>,
mime_type: impl Into<String>,
data_base64: String,
size_bytes: Option<u64>,
) -> Self {
Self {
id: Uuid::new_v4(),
name: Some(name.into()),
description: None,
mime_type: mime_type.into(),
source_path: None,
data_base64: Some(data_base64),
text_content: None,
size_bytes,
preview_lines: None,
}
}
/// Build an attachment from UTF-8 text content.
pub fn from_text(name: Option<String>, mime_type: impl Into<String>, text: String) -> Self {
Self {
id: Uuid::new_v4(),
name,
description: None,
mime_type: mime_type.into(),
source_path: None,
data_base64: None,
text_content: Some(text),
size_bytes: None,
preview_lines: None,
}
}
/// Attach a source path reference to the attachment.
pub fn with_source_path(mut self, path: PathBuf) -> Self {
self.source_path = Some(path);
self
}
/// Set the description metadata for the attachment.
pub fn with_description(mut self, description: impl Into<String>) -> Self {
self.description = Some(description.into());
self
}
/// Provide pre-rendered preview lines for rapid UI display.
pub fn with_preview_lines(mut self, lines: Vec<String>) -> Self {
if lines.is_empty() {
self.preview_lines = None;
} else {
self.preview_lines = Some(lines);
}
self
}
/// Returns true if the attachment MIME type indicates an image.
pub fn is_image(&self) -> bool {
self.mime_type.to_ascii_lowercase().starts_with("image/")
}
/// Accessor for base64 data payloads.
pub fn base64_data(&self) -> Option<&str> {
self.data_base64.as_deref()
}
/// Accessor for inline text payloads.
pub fn text_data(&self) -> Option<&str> {
self.text_content.as_deref()
}
}
impl fmt::Display for Role {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let label = match self {
@@ -168,6 +281,7 @@ impl Message {
metadata: HashMap::new(),
timestamp: std::time::SystemTime::now(),
tool_calls: None,
attachments: Vec::new(),
}
}
@@ -203,6 +317,17 @@ impl Message {
.map(|tc| !tc.is_empty())
.unwrap_or(false)
}
/// Attach rich artifacts to the message.
pub fn with_attachments(mut self, attachments: Vec<MessageAttachment>) -> Self {
self.attachments = attachments;
self
}
/// Return true when the message carries any attachments.
pub fn has_attachments(&self) -> bool {
!self.attachments.is_empty()
}
}
impl Conversation {

3
crates/owlen-tui/Cargo.toml
View File

@@ -45,6 +45,9 @@ serde_json.workspace = true
serde.workspace = true
chrono = { workspace = true }
log = { workspace = true }
base64 = { workspace = true }
mime_guess = { workspace = true }
image = { workspace = true }
[dev-dependencies]
tokio-test = { workspace = true }

620
crates/owlen-tui/src/chat_app.rs
View File

@@ -1,10 +1,13 @@
use anyhow::{Context, Result, anyhow};
use async_trait::async_trait;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use chrono::{DateTime, Local, Utc};
use crossterm::{
event::{KeyEvent, MouseButton, MouseEvent, MouseEventKind},
terminal::{disable_raw_mode, enable_raw_mode},
};
use image::{self, GenericImageView, imageops::FilterType};
use mime_guess;
use owlen_core::Error as CoreError;
use owlen_core::consent::ConsentScope;
use owlen_core::facade::llm_client::LlmClient;
@@ -27,7 +30,9 @@ use owlen_core::{
},
storage::SessionMeta,
theme::Theme,
types::{ChatParameters, ChatResponse, Conversation, ModelInfo, Role, TokenUsage},
types::{
ChatParameters, ChatResponse, Conversation, MessageAttachment, ModelInfo, Role, TokenUsage,
},
ui::{AppState, AutoScroll, FocusedPanel, InputMode, RoleLabelDisplay},
usage::{UsageBand, UsageSnapshot, UsageWindow, WindowMetrics},
};
@@ -116,6 +121,12 @@ const MOUSE_SCROLL_STEP: isize = 3;
const DEFAULT_CONTEXT_WINDOW_TOKENS: u32 = 8_192;
const MAX_QUEUE_ATTEMPTS: u8 = 3;
const THOUGHT_SUMMARY_LIMIT: usize = 5;
const MAX_ATTACHMENT_BYTES: u64 = 8 * 1024 * 1024;
const ATTACHMENT_ASCII_WIDTH: u32 = 24;
const ATTACHMENT_ASCII_HEIGHT: u32 = 12;
const ATTACHMENT_TEXT_PREVIEW_LINES: usize = 12;
const ATTACHMENT_TEXT_PREVIEW_WIDTH: usize = 80;
const ATTACHMENT_INLINE_PREVIEW_LINES: usize = 6;
#[derive(Clone, Copy, Debug, Default)]
pub struct ContextUsage {
@@ -294,6 +305,7 @@ pub(crate) struct LayoutSnapshot {
pub(crate) chat_panel: Option<Rect>,
pub(crate) thinking_panel: Option<Rect>,
pub(crate) actions_panel: Option<Rect>,
pub(crate) attachments_panel: Option<Rect>,
pub(crate) input_panel: Option<Rect>,
pub(crate) system_panel: Option<Rect>,
pub(crate) status_panel: Option<Rect>,
@@ -312,6 +324,7 @@ impl LayoutSnapshot {
chat_panel: None,
thinking_panel: None,
actions_panel: None,
attachments_panel: None,
input_panel: None,
system_panel: None,
status_panel: None,
@@ -368,6 +381,11 @@ impl LayoutSnapshot {
return Some(UiRegion::Actions);
}
}
if let Some(rect) = self.attachments_panel {
if Self::contains(rect, column, row) {
return Some(UiRegion::Attachments);
}
}
if let Some(rect) = self.thinking_panel {
if Self::contains(rect, column, row) {
return Some(UiRegion::Thinking);
@@ -403,6 +421,7 @@ enum UiRegion {
Chat,
Thinking,
Actions,
Attachments,
Input,
System,
Status,
@@ -768,8 +787,12 @@ pub struct ChatApp {
command_palette: CommandPalette, // Command mode state (buffer + suggestions)
resource_catalog: Vec<McpResourceConfig>, // Configured MCP resources for autocompletion
pending_resource_refs: Vec<String>, // Resource references to resolve before send
pending_attachments: Vec<MessageAttachment>, // Attachments staged for the next user turn
attachment_preview_entries: Vec<AttachmentPreviewEntry>,
attachment_preview_selection: usize,
attachment_preview_source: AttachmentPreviewSource,
oauth_flows: HashMap<String, DeviceAuthorization>, // Active OAuth device flows by server
repo_search: RepoSearchState, // Repository search overlay state
repo_search: RepoSearchState, // Repository search overlay state
repo_search_task: Option<JoinHandle<()>>,
repo_search_rx: Option<mpsc::UnboundedReceiver<RepoSearchMessage>>,
repo_search_file_map: HashMap<PathBuf, usize>,
@@ -871,15 +894,17 @@ struct QueuedCommand {
enqueued_at: DateTime<Utc>,
source: QueueSource,
attempts: u8,
attachments: Vec<MessageAttachment>,
}
impl QueuedCommand {
fn new(content: String, source: QueueSource) -> Self {
fn new(content: String, attachments: Vec<MessageAttachment>, source: QueueSource) -> Self {
Self {
content,
enqueued_at: Utc::now(),
source,
attempts: 0,
attachments,
}
}
@@ -892,6 +917,7 @@ impl QueuedCommand {
enqueued_at: active.enqueued_at,
source: QueueSource::Resume,
attempts: active.attempts + 1,
attachments: active.attachments,
})
}
}
@@ -903,16 +929,24 @@ struct ActiveCommand {
source: QueueSource,
attempts: u8,
enqueued_at: DateTime<Utc>,
attachments: Vec<MessageAttachment>,
}
impl ActiveCommand {
fn new(content: String, source: QueueSource, attempts: u8, enqueued_at: DateTime<Utc>) -> Self {
fn new(
content: String,
attachments: Vec<MessageAttachment>,
source: QueueSource,
attempts: u8,
enqueued_at: DateTime<Utc>,
) -> Self {
Self {
response_id: None,
content,
source,
attempts,
enqueued_at,
attachments,
}
}
@@ -991,6 +1025,19 @@ enum MessageSegment {
},
}
#[derive(Debug, Clone)]
pub(crate) struct AttachmentPreviewEntry {
pub(crate) summary: String,
pub(crate) preview_lines: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum AttachmentPreviewSource {
None,
Pending,
Message(Uuid),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum FileOpenDisposition {
Primary,
@@ -1170,6 +1217,10 @@ impl ChatApp {
command_palette: CommandPalette::new(),
resource_catalog: Vec::new(),
pending_resource_refs: Vec::new(),
pending_attachments: Vec::new(),
attachment_preview_entries: Vec::new(),
attachment_preview_selection: 0,
attachment_preview_source: AttachmentPreviewSource::None,
oauth_flows: HashMap::new(),
repo_search: RepoSearchState::new(),
repo_search_task: None,
@@ -2749,14 +2800,19 @@ impl ChatApp {
|| !self.command_queue.is_empty()
}
fn enqueue_submission(&mut self, content: String, source: QueueSource) {
fn enqueue_submission(
&mut self,
content: String,
attachments: Vec<MessageAttachment>,
source: QueueSource,
) {
let trimmed = content.trim();
if trimmed.is_empty() {
if trimmed.is_empty() && attachments.is_empty() {
self.error = Some("Cannot queue empty message".to_string());
return;
}
let entry = QueuedCommand::new(trimmed.to_string(), source);
let entry = QueuedCommand::new(trimmed.to_string(), attachments, source);
self.command_queue.push_back(entry);
let pending = self.command_queue.len();
if self.queue_paused {
@@ -2775,12 +2831,13 @@ impl ChatApp {
fn start_user_turn_internal(
&mut self,
content: String,
attachments: Vec<MessageAttachment>,
source: QueueSource,
attempts: u8,
enqueued_at: DateTime<Utc>,
) {
let message_body = content.trim();
if message_body.is_empty() {
if message_body.is_empty() && attachments.is_empty() {
self.error = Some("Cannot send empty message".to_string());
self.status = "Message discarded".to_string();
return;
@@ -2791,10 +2848,21 @@ impl ChatApp {
references.dedup();
self.pending_resource_refs = references;
let _message_id = self
.controller
.conversation_mut()
.push_user_message(message_body.to_string());
let attachments_for_message = attachments.clone();
let _message_id = if attachments_for_message.is_empty() {
self.controller
.conversation_mut()
.push_user_message(message_body.to_string())
} else {
self.controller
.conversation_mut()
.push_user_message_with_attachments(
message_body.to_string(),
attachments_for_message,
)
};
self.refresh_attachment_gallery();
self.auto_scroll.stick_to_bottom = true;
self.pending_llm_request = true;
@@ -2805,12 +2873,99 @@ impl ChatApp {
self.active_command = Some(ActiveCommand::new(
message_body.to_string(),
attachments,
source,
attempts,
enqueued_at,
));
}
async fn attach_file(&mut self, raw_path: &str) -> Result<()> {
let expanded = shellexpand::tilde(raw_path).into_owned();
let cleaned = expanded.trim();
if cleaned.is_empty() {
return Err(anyhow!("Attachment path cannot be empty"));
}
let path = PathBuf::from(cleaned);
let metadata = tokio::fs::metadata(&path)
.await
.with_context(|| format!("Unable to inspect {}", path.display()))?;
if !metadata.is_file() {
return Err(anyhow!("{} is not a file", path.display()));
}
if metadata.len() > MAX_ATTACHMENT_BYTES {
return Err(anyhow!(format!(
"Attachments are limited to {} (requested {}): {}",
Self::format_attachment_size(MAX_ATTACHMENT_BYTES),
Self::format_attachment_size(metadata.len()),
path.display()
)));
}
let bytes = tokio::fs::read(&path)
.await
.with_context(|| format!("Failed to read {}", path.display()))?;
let mime = mime_guess::from_path(&path).first_or_octet_stream();
let mime_string = mime.essence_str().to_string();
let file_name = path
.file_name()
.and_then(|value| value.to_str())
.unwrap_or("attachment")
.to_string();
let mut attachment =
if mime_string.starts_with("text/") || std::str::from_utf8(&bytes).is_ok() {
let text = String::from_utf8_lossy(&bytes).into_owned();
let mut attachment = MessageAttachment::from_text(
Some(file_name.clone()),
mime_string.clone(),
text.clone(),
);
attachment.size_bytes = Some(metadata.len());
let preview = Self::preview_lines_for_text(&text);
if !preview.is_empty() {
attachment = attachment.with_preview_lines(preview);
}
attachment
} else {
let encoded = BASE64_STANDARD.encode(&bytes);
let mut attachment = MessageAttachment::from_base64(
file_name.clone(),
mime_string.clone(),
encoded,
Some(metadata.len()),
);
if let Some(preview) = Self::preview_lines_for_image(&bytes) {
attachment = attachment.with_preview_lines(preview);
}
attachment
};
attachment.size_bytes = Some(metadata.len());
attachment = attachment
.with_source_path(path.clone())
.with_description(format!(
"{} ({})",
path.display(),
Self::format_attachment_size(metadata.len())
));
self.pending_attachments.push(attachment);
self.refresh_attachment_gallery();
self.status = format!(
"Attached {} ({})",
path.display(),
Self::format_attachment_size(metadata.len())
);
self.error = None;
self.push_toast(
ToastLevel::Info,
format!("Attachment staged: {}", file_name),
);
Ok(())
}
fn start_user_turn_from_queue(&mut self, entry: QueuedCommand) {
let pending = self.command_queue.len();
self.status = if pending == 0 {
@@ -2820,6 +2975,7 @@ impl ChatApp {
};
self.start_user_turn_internal(
entry.content,
entry.attachments,
entry.source,
entry.attempts,
entry.enqueued_at,
@@ -3502,6 +3658,257 @@ impl ChatApp {
references
}
fn refresh_attachment_gallery(&mut self) {
let mut entries = Vec::new();
let mut source = AttachmentPreviewSource::None;
if !self.pending_attachments.is_empty() {
entries = self
.pending_attachments
.iter()
.map(|attachment| self.build_attachment_entry(attachment))
.collect();
source = AttachmentPreviewSource::Pending;
} else if let Some(message) = self.latest_message_with_attachments() {
entries = message
.attachments
.iter()
.map(|attachment| self.build_attachment_entry(attachment))
.collect();
if !entries.is_empty() {
source = AttachmentPreviewSource::Message(message.id);
}
}
if entries.is_empty() {
self.attachment_preview_entries.clear();
self.attachment_preview_selection = 0;
self.attachment_preview_source = AttachmentPreviewSource::None;
} else {
let max_index = entries.len().saturating_sub(1);
self.attachment_preview_selection = self.attachment_preview_selection.min(max_index);
self.attachment_preview_entries = entries;
self.attachment_preview_source = source;
}
}
fn latest_message_with_attachments(&self) -> Option<&owlen_core::types::Message> {
self.controller
.conversation()
.messages
.iter()
.rev()
.find(|message| !message.attachments.is_empty())
}
fn build_attachment_entry(&self, attachment: &MessageAttachment) -> AttachmentPreviewEntry {
let preview_lines = self.generate_attachment_preview_lines(attachment);
let summary = Self::summarize_attachment(attachment);
AttachmentPreviewEntry {
summary,
preview_lines,
}
}
fn generate_attachment_preview_lines(&self, attachment: &MessageAttachment) -> Vec<String> {
if let Some(lines) = attachment.preview_lines.clone() {
return lines;
}
if let Some(text) = attachment.text_data() {
return Self::preview_lines_for_text(text);
}
if attachment.is_image() {
if let Some(data) = attachment.base64_data() {
if let Ok(bytes) = BASE64_STANDARD.decode(data) {
if let Some(lines) = Self::preview_lines_for_image(&bytes) {
return lines;
}
}
} else if let Some(path) = attachment.source_path.as_ref() {
if let Ok(bytes) = fs::read(path) {
if let Some(lines) = Self::preview_lines_for_image(&bytes) {
return lines;
}
}
}
}
Vec::new()
}
pub(crate) fn attachment_preview_height(&self) -> u16 {
if self.attachment_preview_entries.is_empty() {
return 0;
}
let list_height = (self.attachment_preview_entries.len() as u16).min(4);
let preview_lines = self
.attachment_preview_entries
.get(self.attachment_preview_selection)
.map(|entry| entry.preview_lines.len() as u16)
.unwrap_or(0)
.min(ATTACHMENT_ASCII_HEIGHT as u16);
let base = 3u16; // header + padding
(base + list_height + preview_lines).min(ATTACHMENT_ASCII_HEIGHT as u16 + 6)
}
pub(crate) fn attachment_preview_entries(&self) -> &[AttachmentPreviewEntry] {
&self.attachment_preview_entries
}
pub(crate) fn attachment_preview_selection(&self) -> usize {
if self.attachment_preview_entries.is_empty() {
0
} else {
self.attachment_preview_selection
.min(self.attachment_preview_entries.len() - 1)
}
}
pub(crate) fn attachment_preview_source(&self) -> AttachmentPreviewSource {
self.attachment_preview_source
}
pub(crate) fn shift_attachment_selection(&mut self, delta: isize) {
if self.attachment_preview_entries.is_empty() {
self.status = "No attachments to select".to_string();
self.error = None;
return;
}
let len = self.attachment_preview_entries.len() as isize;
if len == 0 {
return;
}
let current = self.attachment_preview_selection as isize;
let mut next = current + delta;
while next < 0 {
next += len;
}
while next >= len {
next -= len;
}
self.attachment_preview_selection = next as usize;
if let Some(entry) = self
.attachment_preview_entries
.get(self.attachment_preview_selection)
{
self.status = format!(
"Viewing attachment {} — {}",
self.attachment_preview_selection + 1,
entry.summary
);
self.error = None;
}
}
fn summarize_attachment(attachment: &MessageAttachment) -> String {
let icon = if attachment.is_image() {
"📷"
} else if attachment
.mime_type
.to_ascii_lowercase()
.starts_with("text/")
{
"📄"
} else {
"📎"
};
let name = attachment
.name
.as_deref()
.unwrap_or(attachment.mime_type.as_str());
let mut parts = Vec::new();
parts.push(format!("{icon} {name}"));
parts.push(attachment.mime_type.clone());
if let Some(size) = attachment.size_bytes {
parts.push(Self::format_attachment_size(size));
}
parts.join(" · ")
}
fn format_attachment_size(bytes: u64) -> String {
const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"];
let mut value = bytes as f64;
let mut index = 0usize;
while value >= 1024.0 && index < UNITS.len() - 1 {
value /= 1024.0;
index += 1;
}
if index == 0 {
format!("{bytes} {}", UNITS[index])
} else {
format!("{value:.1} {}", UNITS[index])
}
}
fn preview_lines_for_text(text: &str) -> Vec<String> {
let mut lines = Vec::new();
for raw in text.lines().take(ATTACHMENT_TEXT_PREVIEW_LINES) {
let trimmed = raw.trim_end();
if trimmed.is_empty() {
lines.push(String::new());
continue;
}
let mut snippet = trimmed
.chars()
.take(ATTACHMENT_TEXT_PREVIEW_WIDTH)
.collect::<String>();
if trimmed.chars().count() > ATTACHMENT_TEXT_PREVIEW_WIDTH {
snippet.push('…');
}
lines.push(snippet);
}
if lines.is_empty() {
lines.push("(empty attachment)".to_string());
}
lines
}
fn preview_lines_for_image(bytes: &[u8]) -> Option<Vec<String>> {
let image = image::load_from_memory(bytes).ok()?;
let (width, height) = image.dimensions();
let mut lines = Vec::new();
lines.push(format!("{width} × {height} px"));
let target_width = ATTACHMENT_ASCII_WIDTH;
let target_height = ATTACHMENT_ASCII_HEIGHT;
let scale = (target_width as f32 / width as f32)
.min(target_height as f32 / height as f32)
.clamp(0.05, 1.0);
let scaled_width = (width as f32 * scale).max(1.0).round() as u32;
let scaled_height = (height as f32 * scale).max(1.0).round() as u32;
let resized = image
.resize_exact(
scaled_width.max(1),
scaled_height.max(1),
FilterType::Triangle,
)
.to_luma8();
const PALETTE: [char; 10] = [' ', '.', ':', '-', '=', '+', '*', '#', '%', '@'];
for y in 0..resized.height() {
let mut row = String::with_capacity((resized.width() as usize) * 2);
for x in 0..resized.width() {
let luminance = resized.get_pixel(x, y)[0] as usize;
let idx = luminance * (PALETTE.len() - 1) / 255;
let ch = PALETTE[idx];
row.push(ch);
row.push(ch);
}
lines.push(row);
}
Some(lines)
}
pub(crate) fn display_name_for_model(model: &ModelInfo) -> String {
let base = {
let trimmed = model.name.trim();
@@ -3601,11 +4008,17 @@ impl ChatApp {
}
}
fn message_content_hash(role: &Role, content: &str, tool_signature: &str) -> u64 {
fn message_content_hash(
role: &Role,
content: &str,
tool_signature: &str,
attachment_signature: &str,
) -> u64 {
let mut hasher = DefaultHasher::new();
role.to_string().hash(&mut hasher);
content.hash(&mut hasher);
tool_signature.hash(&mut hasher);
attachment_signature.hash(&mut hasher);
hasher.finish()
}
@@ -3723,7 +4136,7 @@ impl ChatApp {
syntax_highlighting,
render_markdown,
} = ctx;
let (message_id, role, raw_content, timestamp, tool_calls, tool_result_id) = {
let (message_id, role, raw_content, timestamp, tool_calls, tool_result_id, attachments) = {
let conversation = self.conversation();
let message = &conversation.messages[message_index];
(
@@ -3737,6 +4150,7 @@ impl ChatApp {
.get("tool_call_id")
.and_then(|value| value.as_str())
.map(|value| value.to_string()),
message.attachments.clone(),
)
};
@@ -3760,7 +4174,13 @@ impl ChatApp {
names.join("|")
})
.unwrap_or_default();
let content_hash = Self::message_content_hash(&role, &content, &tool_signature);
let attachment_signature = attachments
.iter()
.map(|attachment| attachment.id.to_string())
.collect::<Vec<_>>()
.join("|");
let content_hash =
Self::message_content_hash(&role, &content, &tool_signature, &attachment_signature);
if !is_streaming {
if let Some(entry) = self.message_line_cache.get(&message_id) {
@@ -3898,6 +4318,40 @@ impl ChatApp {
}
}
if !attachments.is_empty() {
if !rendered.is_empty() {
rendered.push(Line::from(vec![Span::raw("")]));
}
for (idx, attachment) in attachments.iter().enumerate() {
let summary = Self::summarize_attachment(attachment);
let header_line = Line::from(vec![
Span::styled("┆ ", Style::default().fg(theme.placeholder)),
Span::styled(
format!("Attachment {}:", idx + 1),
Style::default()
.fg(theme.placeholder)
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(summary, content_style),
]);
rendered.push(header_line);
let preview_lines = self.generate_attachment_preview_lines(attachment);
for preview in preview_lines
.into_iter()
.take(ATTACHMENT_INLINE_PREVIEW_LINES)
{
rendered.push(Line::from(vec![Span::styled(
format!(" {}", preview),
Style::default()
.fg(theme.placeholder)
.add_modifier(Modifier::DIM),
)]));
}
}
}
let markers =
Self::message_tool_markers(&role, tool_calls.as_ref(), tool_result_id.as_deref());
let formatted_timestamp = if self.show_message_timestamps {
@@ -7909,6 +8363,98 @@ impl ChatApp {
}
}
}
"attach" => {
if args.is_empty() {
self.error = Some("Usage: :attach <path>".to_string());
} else {
let path_arg = args.join(" ");
match self.attach_file(&path_arg).await {
Ok(()) => {}
Err(err) => {
self.status = "Attachment failed".to_string();
self.error = Some(err.to_string());
}
}
}
}
"attachments" => {
let action = args.first().map(|s| s.to_ascii_lowercase());
match action.as_deref() {
Some("clear") => {
if self.pending_attachments.is_empty() {
self.status =
"No staged attachments to clear".to_string();
} else {
self.pending_attachments.clear();
self.refresh_attachment_gallery();
self.status =
"Cleared staged attachments".to_string();
}
self.error = None;
}
Some("next") | Some("n") => {
self.shift_attachment_selection(1);
}
Some("prev") | Some("previous") | Some("p") => {
self.shift_attachment_selection(-1);
}
Some("remove") => {
if args.len() < 2 {
self.error = Some(
"Usage: :attachments remove <index>"
.to_string(),
);
} else if let Ok(index) = args[1].parse::<usize>() {
if index == 0
|| index > self.pending_attachments.len()
{
self.error = Some(format!(
"Attachment index {} out of range",
index
));
} else {
self.pending_attachments.remove(index - 1);
self.refresh_attachment_gallery();
self.status =
format!("Removed attachment {}", index);
self.error = None;
}
} else {
self.error = Some(
"Attachment index must be a positive integer"
.to_string(),
);
}
}
None => {
if self.pending_attachments.is_empty() {
self.status = "No staged attachments".to_string();
} else {
let list = self
.pending_attachments
.iter()
.enumerate()
.map(|(idx, attachment)| {
format!(
"{}. {}",
idx + 1,
Self::summarize_attachment(attachment)
)
})
.collect::<Vec<_>>()
.join(" • ");
self.status = format!("Staged attachments: {list}");
}
self.error = None;
}
Some(other) => {
self.error = Some(format!(
"Unknown attachments command '{}'. Use :attachments, :attachments clear, or :attachments remove <index>.",
other
));
}
}
}
"files" | "explorer" => {
if !self.is_code_mode() {
self.status =
@@ -8096,6 +8642,8 @@ impl ChatApp {
self.chat_line_offset = 0;
self.auto_scroll = AutoScroll::default();
self.clear_new_message_alert();
self.pending_attachments.clear();
self.refresh_attachment_gallery();
self.status = "Conversation cleared".to_string();
}
"session" => {
@@ -9739,6 +10287,10 @@ impl ChatApp {
self.thinking_scroll.on_user_scroll(amount, viewport);
}
}
UiRegion::Attachments => {
let delta = if amount > 0 { 1 } else { -1 };
self.shift_attachment_selection(delta);
}
UiRegion::Code => {
if self.focus_panel(FocusedPanel::Code) {
let viewport = self.code_view_viewport_height().max(1);
@@ -9777,6 +10329,17 @@ impl ChatApp {
self.set_input_mode(InputMode::Normal);
}
}
UiRegion::Attachments => {
if let Some(rect) = self.last_layout.attachments_panel {
if row > rect.y + 1 {
let list_index = row.saturating_sub(rect.y + 1) as usize;
if list_index < self.attachment_preview_entries.len() {
self.attachment_preview_selection = list_index;
}
}
}
self.shift_attachment_selection(0);
}
UiRegion::Code => {
if self.focus_panel(FocusedPanel::Code) {
self.set_input_mode(InputMode::Normal);
@@ -10083,6 +10646,8 @@ impl ChatApp {
self.mark_active_command_succeeded();
}
}
self.refresh_attachment_gallery();
}
SessionEvent::StreamError {
message_id,
@@ -10119,6 +10684,7 @@ impl ChatApp {
.conversation_mut()
.push_assistant_message(answer);
self.notify_new_activity();
self.refresh_attachment_gallery();
self.agent_running = false;
self.agent_mode = false;
self.agent_actions = None;
@@ -12081,17 +12647,21 @@ impl ChatApp {
fn send_user_message_and_request_response(&mut self) {
let raw = self.controller.input_buffer_mut().commit_to_history();
let content = raw.trim().to_string();
if content.is_empty() {
let attachments = std::mem::take(&mut self.pending_attachments);
if content.is_empty() && attachments.is_empty() {
self.error = Some("Cannot send empty message".to_string());
return;
}
self.refresh_attachment_gallery();
if self.should_queue_submission() {
self.enqueue_submission(content, QueueSource::User);
self.enqueue_submission(content, attachments, QueueSource::User);
return;
}
self.start_user_turn_internal(content, QueueSource::User, 0, Utc::now());
self.start_user_turn_internal(content, attachments, QueueSource::User, 0, Utc::now());
}
pub fn has_active_generation(&self) -> bool {
@@ -12461,7 +13031,7 @@ impl ChatApp {
use owlen_core::mcp::remote_client::RemoteMcpClient;
use std::sync::Arc;
// Get the last user message
// Get the last user message (including attachments)
let user_message = self
.controller
.conversation()
@@ -12469,8 +13039,11 @@ impl ChatApp {
.iter()
.rev()
.find(|m| m.role == owlen_core::types::Role::User)
.map(|m| m.content.clone())
.unwrap_or_default();
.cloned()
.unwrap_or_else(|| owlen_core::types::Message::user(String::new()));
let user_prompt = user_message.content.clone();
let user_attachments = user_message.attachments.clone();
let profile = match self.active_agent_profile().cloned() {
Some(profile) => profile,
@@ -12535,7 +13108,10 @@ impl ChatApp {
let executor = AgentExecutor::new(provider, mcp_client, config);
// Run agent
match executor.run(user_message).await {
match executor
.run_with_attachments(user_prompt, user_attachments)
.await
{
Ok(result) => {
let message_id = self
.controller

18
crates/owlen-tui/src/commands/mod.rs
View File

@@ -163,6 +163,24 @@ const COMMANDS: &[CommandDescriptor] = &[
keybinding: None,
preview: None,
},
CommandDescriptor {
keywords: &["attach"],
description: "Attach a file to the next user turn",
category: CommandCategory::Conversation,
modes: &["Command"],
tags: &["file", "attachment", "multimodal"],
keybinding: None,
preview: None,
},
CommandDescriptor {
keywords: &["attachments"],
description: "Manage staged attachments (list, next, clear, remove)",
category: CommandCategory::Conversation,
modes: &["Command"],
tags: &["attachment", "preview", "queue"],
keybinding: None,
preview: None,
},
CommandDescriptor {
keywords: &["session save"],
description: "Save the current conversation",

120
crates/owlen-tui/src/ui.rs
View File

@@ -15,8 +15,8 @@ use unicode_segmentation::UnicodeSegmentation;
use unicode_width::UnicodeWidthStr;
use crate::chat_app::{
AdaptiveLayout, ChatApp, ContextUsage, GaugeKey, GuidanceOverlay, LayoutSnapshot,
MIN_MESSAGE_CARD_WIDTH, MessageRenderContext, PanePulse,
AdaptiveLayout, AttachmentPreviewSource, ChatApp, ContextUsage, GaugeKey, GuidanceOverlay,
LayoutSnapshot, MIN_MESSAGE_CARD_WIDTH, MessageRenderContext, PanePulse,
};
use crate::glass::{GlassPalette, blend_color, gradient_color};
use crate::highlight;
@@ -33,6 +33,7 @@ use owlen_core::{config::LayerSettings, theme::Theme};
use textwrap::wrap;
const APP_VERSION: &str = env!("CARGO_PKG_VERSION");
const INLINE_ATTACHMENT_PREVIEW_LINES: usize = 6;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ProgressBand {
@@ -1265,6 +1266,11 @@ pub fn render_chat(frame: &mut Frame<'_>, app: &mut ChatApp) {
let mut constraints = vec![Constraint::Min(8)]; // Messages
let attachments_height = app.attachment_preview_height();
if attachments_height > 0 {
constraints.push(Constraint::Length(attachments_height)); // Attachments
}
if thinking_height > 0 {
constraints.push(Constraint::Length(thinking_height)); // Thinking
}
@@ -1292,6 +1298,14 @@ pub fn render_chat(frame: &mut Frame<'_>, app: &mut ChatApp) {
render_messages(frame, layout[idx], app);
idx += 1;
if attachments_height > 0 {
snapshot.attachments_panel = Some(layout[idx]);
render_attachment_preview(frame, layout[idx], app);
idx += 1;
} else {
snapshot.attachments_panel = None;
}
if thinking_height > 0 {
snapshot.thinking_panel = Some(layout[idx]);
render_thinking(frame, layout[idx], app);
@@ -2723,6 +2737,108 @@ fn render_thinking(frame: &mut Frame<'_>, area: Rect, app: &mut ChatApp) {
}
}
fn render_attachment_preview(frame: &mut Frame<'_>, area: Rect, app: &mut ChatApp) {
let entries = app.attachment_preview_entries();
if entries.is_empty() {
return;
}
let theme = app.theme().clone();
let reduced = app.is_reduced_chrome();
let horizontal_padding = if reduced { 1 } else { 2 };
let vertical_padding = if reduced { 0 } else { 1 };
let palette = GlassPalette::for_theme_with_mode(&theme, reduced, app.layer_settings());
let title = match app.attachment_preview_source() {
AttachmentPreviewSource::Pending => format!("Staged Attachments ({})", entries.len()),
AttachmentPreviewSource::Message(_) => {
format!("Message Attachments ({})", entries.len())
}
AttachmentPreviewSource::None => format!("Attachments ({})", entries.len()),
};
let block = Block::default()
.borders(Borders::NONE)
.padding(Padding::new(
horizontal_padding,
horizontal_padding,
vertical_padding,
vertical_padding,
))
.style(Style::default().bg(palette.active).fg(theme.text))
.title(title)
.title_style(Style::default().fg(theme.pane_header_active));
let selected = app
.attachment_preview_selection()
.min(entries.len().saturating_sub(1));
let mut lines: Vec<Line> = Vec::new();
for (idx, entry) in entries.iter().enumerate() {
let index_label = format!("{}. ", idx + 1);
let mut spans = Vec::new();
spans.push(Span::styled(
index_label,
Style::default().fg(theme.placeholder),
));
let style = if idx == selected {
Style::default()
.fg(theme.assistant_message_role)
.add_modifier(Modifier::BOLD)
} else {
Style::default().fg(theme.text)
};
spans.push(Span::styled(entry.summary.clone(), style));
lines.push(Line::from(spans));
}
if let Some(entry) = entries.get(selected) {
if !entry.preview_lines.is_empty() {
lines.push(Line::from(vec![Span::styled(
"",
Style::default()
.fg(theme.placeholder)
.add_modifier(Modifier::DIM),
)]));
for preview in entry
.preview_lines
.iter()
.take(INLINE_ATTACHMENT_PREVIEW_LINES)
{
lines.push(Line::from(vec![Span::styled(
preview.clone(),
Style::default()
.fg(theme.placeholder)
.add_modifier(Modifier::DIM),
)]));
}
}
}
lines.push(Line::from(vec![Span::styled(
"Commands: :attachments next · :attachments prev · :attachments remove <n>",
Style::default()
.fg(theme.placeholder)
.add_modifier(Modifier::DIM),
)]));
if lines.is_empty() {
lines.push(Line::from(vec![Span::styled(
"No attachment details available",
Style::default()
.fg(theme.placeholder)
.add_modifier(Modifier::DIM),
)]));
}
let paragraph = Paragraph::new(lines)
.block(block)
.wrap(Wrap { trim: false })
.style(Style::default().bg(palette.active).fg(theme.text));
frame.render_widget(paragraph, area);
}
// Render a panel displaying the latest ReAct agent actions (thought/action/observation).
// Color-coded: THOUGHT (blue), ACTION (yellow), OBSERVATION (green)
fn render_agent_actions(frame: &mut Frame<'_>, area: Rect, app: &mut ChatApp) {