From c17262cffc32242adfeb10fd0795cab7b4370db0 Mon Sep 17 00:00:00 2001 From: Daniel Katulevskiy Date: Thu, 29 Oct 2026 06:15:25 -0700 Subject: [PATCH 01/81] fix: preview images embedded in tool results Tool chips named a scratch path outside the attachment jail, so the preview read failed and the chip showed Image unavailable. Image bytes already present on the tool result are now stored in the profile uploads directory and the chip previews that copy. A subagent image note about the displayed size was folded as a steer and the pixels were dropped. That note is ignored, and the same stored preview is used on the subagent transcript. --- crates/client/src/demo/transcripts.rs | 1 + crates/client/src/session/mod.rs | 1 + crates/client/src/subagents.rs | 1 + crates/doc/examples/gen_fixture.rs | 2 + crates/doc/src/parts.rs | 75 ++++ crates/doc/src/rebuild.rs | 1 + crates/doc/src/schema.rs | 78 ++++ crates/engine/src/registry.rs | 4 + crates/engine/src/runner.rs | 1 + crates/engine/src/sessions.rs | 350 +++++++++++++-- crates/engine/src/uploads.rs | 27 ++ crates/engine/tests/e2e.rs | 7 + crates/engine/tests/side_chats.rs | 1 + crates/engine/tests/subagent_idle_reap.rs | 2 + crates/engine/tests/turn_quiesce.rs | 4 + crates/harness/src/acp/normalize.rs | 10 + crates/harness/src/acp/subagent.rs | 2 + crates/harness/src/acp/system_message.rs | 2 + crates/harness/src/claude/normalize.rs | 459 +++++++++++++++++--- crates/harness/src/claude/wire.rs | 7 + crates/harness/src/codex/normalize.rs | 16 + crates/harness/src/cursor/mod.rs | 2 + crates/harness/src/mock.rs | 12 + crates/harness/src/muse/normalize.rs | 6 + crates/harness/src/opencode/mod.rs | 2 + crates/harness/src/pi/normalize.rs | 2 + crates/harness/tests/claude.rs | 6 + crates/harness/tests/codex.rs | 14 + crates/harness/tests/cursor.rs | 2 + crates/mcp/src/transcript.rs | 1 + crates/mobile/src/layout/tests.rs | 2 + crates/proto/src/agent.rs | 15 + crates/ui/examples/subagent-fold-fixture.rs | 1 + crates/ui/src/shell/session_lanes.rs | 1 + crates/ui/src/state.rs | 1 + crates/ui/src/todo_panel.rs | 2 + crates/ui/src/tool_images.rs | 45 ++ crates/ui/src/transcript.rs | 6 +- docs/image-preview.md | 2 +- 39 files changed, 1057 insertions(+), 116 deletions(-) diff --git a/crates/client/src/demo/transcripts.rs b/crates/client/src/demo/transcripts.rs index 67e4c1d4d..dfa0ecff0 100644 --- a/crates/client/src/demo/transcripts.rs +++ b/crates/client/src/demo/transcripts.rs @@ -38,6 +38,7 @@ pub(crate) fn tool(id: &str, call: ToolCall, is_error: bool, output: Option<&str subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), } } diff --git a/crates/client/src/session/mod.rs b/crates/client/src/session/mod.rs index 94cd37309..31ee65f83 100644 --- a/crates/client/src/session/mod.rs +++ b/crates/client/src/session/mod.rs @@ -908,6 +908,7 @@ fn refresh_setup_placeholder(st: &mut CoreState, cloud_booting: bool) -> bool { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }], created_at: created_at + 1, device_id: String::new(), diff --git a/crates/client/src/subagents.rs b/crates/client/src/subagents.rs index bebed4156..72a39ceaa 100644 --- a/crates/client/src/subagents.rs +++ b/crates/client/src/subagents.rs @@ -285,6 +285,7 @@ mod tests { subagent_ref: status.map(|_| format!("chat--sub--{id}")), subagent_status: status, subagent_tail: Some("Reading veil.rs".into()), + preview_paths: Vec::new(), } } diff --git a/crates/doc/examples/gen_fixture.rs b/crates/doc/examples/gen_fixture.rs index 34821a16e..3b9ceea30 100644 --- a/crates/doc/examples/gen_fixture.rs +++ b/crates/doc/examples/gen_fixture.rs @@ -55,6 +55,8 @@ fn main() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ); fold_event_into_parts( diff --git a/crates/doc/src/parts.rs b/crates/doc/src/parts.rs index ce42397e1..e8bd7f003 100644 --- a/crates/doc/src/parts.rs +++ b/crates/doc/src/parts.rs @@ -201,6 +201,12 @@ pub enum MessagePart { /// its tagged text deltas (capped; display-only). #[serde(default, skip_serializing_if = "Option::is_none")] subagent_tail: Option, + /// Uploads copies of images embedded in this tool's result. When + /// this is non-empty the chip previews these paths instead of files + /// the call named, which may sit outside the attachment jail. + /// Empty on older parts and on results that carried no image bytes. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + preview_paths: Vec, }, #[serde(rename_all = "camelCase")] Input { @@ -277,6 +283,7 @@ impl MessagePart { output, diff, diff_stats, + preview_paths, .. } => { serde_json::to_vec(call).map_or(0, |v| v.len()) @@ -287,6 +294,7 @@ impl MessagePart { + diff_stats .as_ref() .map_or(0, |s| serde_json::to_vec(s).map_or(0, |v| v.len())) + + preview_paths.iter().map(String::len).sum::() } MessagePart::Input { questions, answers, .. @@ -413,6 +421,7 @@ pub fn fold_event_into_parts(out: &mut Vec, event: &AgentEvent) { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }); } } @@ -421,6 +430,8 @@ pub fn fold_event_into_parts(out: &mut Vec, event: &AgentEvent) { is_error, output, diff, + preview_paths, + .. } => { for p in out.iter_mut() { if let MessagePart::Tool { @@ -431,6 +442,7 @@ pub fn fold_event_into_parts(out: &mut Vec, event: &AgentEvent) { diff: diff_slot, output_bytes, diff_stats, + preview_paths: stored, .. } = p && pid == id @@ -449,6 +461,11 @@ pub fn fold_event_into_parts(out: &mut Vec, event: &AgentEvent) { *output_bytes = None; *diff_slot = None; *diff_stats = diff.as_ref().map(|d| vec![diff_stat(d)]); + // A later result with no images must not wipe a preview + // already stored for this chip. + if !preview_paths.is_empty() { + *stored = preview_paths.clone(); + } } } } @@ -999,6 +1016,8 @@ mod tests { is_error: true, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ); match &parts[0] { @@ -1012,6 +1031,53 @@ mod tests { } } + #[test] + fn tool_result_preview_paths_stick_and_an_empty_result_does_not_wipe_them() { + let mut parts = Vec::new(); + fold_event_into_parts( + &mut parts, + &AgentEvent::ToolCall { + id: "t".into(), + call: ToolCall::ReadFile { + path: "/tmp/shots/1.png".into(), + }, + }, + ); + fold_event_into_parts( + &mut parts, + &AgentEvent::ToolResult { + id: "t".into(), + is_error: false, + output: None, + diff: None, + preview_paths: vec!["/uploads/abc-image.png".into()], + image_payloads: Vec::new(), + }, + ); + fold_event_into_parts( + &mut parts, + &AgentEvent::ToolResult { + id: "t".into(), + is_error: false, + output: None, + diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), + }, + ); + match &parts[0] { + MessagePart::Tool { + preview_paths, + resolved, + .. + } => { + assert!(*resolved); + assert_eq!(preview_paths, &vec!["/uploads/abc-image.png".to_string()]); + } + other => panic!("unexpected {other:?}"), + } + } + #[test] fn sanitize_strips_heavy_inputs_and_is_idempotent() { let call = ToolCall::WriteFile { @@ -1117,6 +1183,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }, ]; let chunks = split_parts(&parts); @@ -1226,6 +1293,8 @@ mod tests { old_text: Some("a\n".into()), new_text: "b\n".into(), }), + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ); match &parts[0] { @@ -1282,6 +1351,8 @@ mod tests { old_text: None, new_text: "x\n".into(), }), + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ); apply_sidecar_refs("chat-9", &mut parts); @@ -1436,6 +1507,8 @@ mod tests { is_error: false, output: Some(" \n".into()), diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }), None, "blank output uploads nothing" @@ -1445,6 +1518,8 @@ mod tests { is_error: true, output: Some("full output".into()), diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }) .unwrap(); assert_eq!(payload.part_id, "t"); diff --git a/crates/doc/src/rebuild.rs b/crates/doc/src/rebuild.rs index 233b512e5..fa0bc6a38 100644 --- a/crates/doc/src/rebuild.rs +++ b/crates/doc/src/rebuild.rs @@ -183,6 +183,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }, ], created_at: 5, diff --git a/crates/doc/src/schema.rs b/crates/doc/src/schema.rs index fe60800c3..ed5b184ec 100644 --- a/crates/doc/src/schema.rs +++ b/crates/doc/src/schema.rs @@ -142,6 +142,9 @@ struct DocPartJson { /// One-line live tail of the subagent's output (additive). #[serde(default, skip_serializing_if = "Option::is_none")] subagent_tail: Option, + /// Uploads copies of images embedded in a tool result (additive). + #[serde(default, skip_serializing_if = "Option::is_none")] + preview_paths: Option>, } /// App parts → doc part json (mirror of `toDocParts`). @@ -194,6 +197,7 @@ fn to_doc_part(part: &MessagePart) -> Result { subagent_ref, subagent_status, subagent_tail, + preview_paths, } => DocPartJson { id: id.clone(), kind: "tool".into(), @@ -217,6 +221,7 @@ fn to_doc_part(part: &MessagePart) -> Result { .to_owned() }), subagent_tail: subagent_tail.clone(), + preview_paths: (!preview_paths.is_empty()).then(|| preview_paths.clone()), ..Default::default() }, MessagePart::Input { @@ -303,6 +308,7 @@ fn from_doc_part(p: DocPartJson) -> MessagePart { _ => None, }), subagent_tail: p.subagent_tail, + preview_paths: p.preview_paths.unwrap_or_default(), }, None => MessagePart::Text { id: p.id, @@ -1075,6 +1081,18 @@ fn push_part(parts: &LoroList, part: &MessagePart) -> Result<(), DocError> { if let Some(subagent_tail) = &doc_part.subagent_tail { map.insert("subagentTail", subagent_tail.as_str())?; } + if let Some(preview_paths) = &doc_part.preview_paths { + map.insert( + "previewPaths", + loro_value_from_json(&serde_json::Value::Array( + preview_paths + .iter() + .cloned() + .map(serde_json::Value::String) + .collect(), + )), + )?; + } Ok(()) } @@ -1239,6 +1257,10 @@ fn salvage_part(part: &serde_json::Value, entry_id: &str, ix: usize) -> Option Result<(), DocError> if let Some(subagent_tail) = &doc_part.subagent_tail { map.insert("subagentTail", subagent_tail.as_str())?; } + if let Some(preview_paths) = &doc_part.preview_paths { + map.insert( + "previewPaths", + loro_value_from_json(&serde_json::Value::Array( + preview_paths + .iter() + .cloned() + .map(serde_json::Value::String) + .collect(), + )), + )?; + } if let Some(text) = &doc_part.text { // Defensive path only — the fold never rewrites earlier text. if let Some(loro::ValueOrContainer::Container(loro::Container::Text(t))) = map.get("text") { @@ -1843,6 +1877,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }; let doc = SessionDoc::init("c1").unwrap(); let mut w = SegmentWriter::begin(&doc, "e1", "dev", 1).unwrap(); @@ -1907,6 +1942,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }; w.sync(std::slice::from_ref(&part)).unwrap(); if let MessagePart::Tool { @@ -1959,6 +1995,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }; let parts = vec![ tool( @@ -2140,6 +2177,42 @@ mod tests { assert_eq!(ea.len(), 2); // one insert from each peer, converged in the same order } + #[test] + fn tool_preview_paths_round_trip_through_the_doc() { + let doc = SessionDoc::init("previews").unwrap(); + let mut writer = SegmentWriter::begin(&doc, "a1", "dev", 1).unwrap(); + let mut folded = Vec::new(); + fold_event_into_parts( + &mut folded, + &AgentEvent::ToolCall { + id: "t".into(), + call: ToolCall::ReadFile { + path: "/tmp/shots/1.png".into(), + }, + }, + ); + writer.sync(&folded).unwrap(); + fold_event_into_parts( + &mut folded, + &AgentEvent::ToolResult { + id: "t".into(), + is_error: false, + output: None, + diff: None, + preview_paths: vec!["/uploads/abc-image.png".into()], + image_payloads: Vec::new(), + }, + ); + writer.finish(&folded, MessageStatus::Complete).unwrap(); + let entries = doc.read_entries().unwrap(); + match &entries[0].parts[0] { + MessagePart::Tool { preview_paths, .. } => { + assert_eq!(preview_paths, &vec!["/uploads/abc-image.png".to_string()]); + } + other => panic!("unexpected {other:?}"), + } + } + #[test] fn segment_writer_streams_text_incrementally() { let doc = SessionDoc::init("chat-1").unwrap(); @@ -2167,6 +2240,8 @@ mod tests { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ); writer.sync(&folded).unwrap(); @@ -2284,6 +2359,8 @@ mod tests { old_text: Some("old\n".into()), new_text: "new\n".into(), }), + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ); crate::parts::apply_sidecar_refs("chat-2", &mut folded); @@ -2347,6 +2424,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }], created_at: 1, device_id: "dev-a".into(), diff --git a/crates/engine/src/registry.rs b/crates/engine/src/registry.rs index ac37af007..1438a1056 100644 --- a/crates/engine/src/registry.rs +++ b/crates/engine/src/registry.rs @@ -763,6 +763,8 @@ fn full_registry() -> HarnessRegistry { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, AgentEvent::ToolCall { id: "mock-tool-2".into(), @@ -776,6 +778,8 @@ fn full_registry() -> HarnessRegistry { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, AgentEvent::TextDelta { text: "The `SegmentWriter` appends into `LoroText` so the oplog stays RLE-merged:\n\n```rust\nfolded = fold_event_into_parts(&folded, &event);\nwriter.sync(&folded)?; // 120ms coalesced commits\n```\n\nSynced to every device through the session room. *Mock harness reporting in.*".into(), diff --git a/crates/engine/src/runner.rs b/crates/engine/src/runner.rs index d81165d8e..ea27e2a0a 100644 --- a/crates/engine/src/runner.rs +++ b/crates/engine/src/runner.rs @@ -462,6 +462,7 @@ fn setup_parts( subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), } }) .collect() diff --git a/crates/engine/src/sessions.rs b/crates/engine/src/sessions.rs index 413a0e657..43baa505a 100644 --- a/crates/engine/src/sessions.rs +++ b/crates/engine/src/sessions.rs @@ -1889,6 +1889,88 @@ impl SessionsEngine { } impl Inner { + /// Store image bytes a tool result already carried. The model-supplied + /// path is never opened. A bad payload leaves no preview and does not + /// fail the turn. The same key overwrites the same uploads file. + async fn store_tool_result_images( + &self, + chat_id: &str, + parents: &[String], + tool_id: &str, + payloads: Vec, + ) -> Vec { + let Some((uploads, _)) = self.generated_images.get() else { + tracing::warn!( + chat = %chat_id, + tool = %tool_id, + "tool image dropped: uploads are not configured" + ); + return Vec::new(); + }; + let uploads = uploads.clone(); + let chat_id = chat_id.to_owned(); + let parents = parents.to_vec(); + let tool_id = tool_id.to_owned(); + let chat_for_log = chat_id.clone(); + let tool_for_log = tool_id.clone(); + tokio::task::spawn_blocking(move || { + use base64::Engine as _; + use sha2::{Digest, Sha256}; + let mut paths = Vec::new(); + for (index, payload) in payloads.into_iter().enumerate().take(4) { + let bytes = match base64::engine::general_purpose::STANDARD + .decode(payload.data_base64.trim()) + { + Ok(bytes) => bytes, + Err(err) => { + tracing::warn!( + chat = %chat_id, + tool = %tool_id, + index, + error = %err, + "tool image rejected" + ); + continue; + } + }; + let mut key = String::new(); + key.push_str(&chat_id); + for parent in &parents { + key.push('\0'); + key.push_str(parent); + } + key.push('\0'); + key.push_str(&tool_id); + key.push('\0'); + key.push_str(&index.to_string()); + let id = format!("{:x}", Sha256::digest(key.as_bytes())); + match uploads.store_shown_image(&id, &payload.name, &bytes) { + Ok(image) => paths.push(image.path), + Err(err) => { + tracing::warn!( + chat = %chat_id, + tool = %tool_id, + index, + error = %err, + "tool image rejected" + ); + } + } + } + paths + }) + .await + .unwrap_or_else(|err| { + tracing::warn!( + chat = %chat_for_log, + tool = %tool_for_log, + error = %err, + "tool image store failed" + ); + Vec::new() + }) + } + /// Journal + broadcast one event (the two unconditional legs of the pipeline). /// Sanitize before ANY journal/publish path, including nested subagent events. async fn prepare_generated_image( @@ -1907,56 +1989,87 @@ impl Inner { parents.push(parent_tool_use_id); leaf = *event; } - let events = if let AgentEvent::GeneratedImage { id, path, .. } = leaf { - let key = std::iter::once(chat_id) - .chain(parents.iter().map(String::as_str)) - .chain(std::iter::once(id.as_str())) - .collect::>() - .join("\0"); - if seen.contains(&key) { - return vec![]; - } - let result = if let Some((uploads, allowed_root)) = self.generated_images.get() { - let uploads = uploads.clone(); - let root = allowed_root.clone(); - let stable_key = key.clone(); - tokio::task::spawn_blocking(move || { - uploads.import_generated_image(std::path::Path::new(&path), &root, &stable_key) - }) - .await - .unwrap_or_else(|err| Err(EngineError::Other(err.to_string()))) - } else { - Err(EngineError::Other( - "Generated image intake is not configured".into(), - )) - }; - match result { - Ok(image) => { - seen.insert(key); - vec![AgentEvent::GeneratedImage { - id, - path: image.path, - name: image.name, - mime_type: image.mime_type, - }] + let events = match leaf { + AgentEvent::GeneratedImage { id, path, .. } => { + let key = std::iter::once(chat_id) + .chain(parents.iter().map(String::as_str)) + .chain(std::iter::once(id.as_str())) + .collect::>() + .join("\0"); + if seen.contains(&key) { + return vec![]; } - Err(err) => { - tracing::warn!(chat = %chat_id, error = %err, "generated image import failed"); - vec![ - AgentEvent::ToolResult { - id: id.strip_suffix(":image").unwrap_or(&id).into(), - is_error: true, - output: None, - diff: None, - }, - AgentEvent::Error { - message: "Generated image unavailable".into(), - }, - ] + let result = if let Some((uploads, allowed_root)) = self.generated_images.get() { + let uploads = uploads.clone(); + let root = allowed_root.clone(); + let stable_key = key.clone(); + tokio::task::spawn_blocking(move || { + uploads.import_generated_image( + std::path::Path::new(&path), + &root, + &stable_key, + ) + }) + .await + .unwrap_or_else(|err| Err(EngineError::Other(err.to_string()))) + } else { + Err(EngineError::Other( + "Generated image intake is not configured".into(), + )) + }; + match result { + Ok(image) => { + seen.insert(key); + vec![AgentEvent::GeneratedImage { + id, + path: image.path, + name: image.name, + mime_type: image.mime_type, + }] + } + Err(err) => { + tracing::warn!(chat = %chat_id, error = %err, "generated image import failed"); + vec![ + AgentEvent::ToolResult { + id: id.strip_suffix(":image").unwrap_or(&id).into(), + is_error: true, + output: None, + diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), + }, + AgentEvent::Error { + message: "Generated image unavailable".into(), + }, + ] + } } } - } else { - vec![leaf] + AgentEvent::ToolResult { + id, + is_error, + output, + diff, + preview_paths, + image_payloads, + } if !image_payloads.is_empty() => { + let stored = self + .store_tool_result_images(chat_id, &parents, &id, image_payloads) + .await; + vec![AgentEvent::ToolResult { + id, + is_error, + output, + diff, + preview_paths: if stored.is_empty() { + preview_paths + } else { + stored + }, + image_payloads: Vec::new(), + }] + } + other => vec![other], }; events .into_iter() @@ -3275,6 +3388,7 @@ fn render_parts(parts: &[MessagePart]) -> Vec { subagent_ref, subagent_status, subagent_tail, + preview_paths, } => MessagePart::Tool { id: id.clone(), call: sanitize_tool_call(call), @@ -3293,6 +3407,7 @@ fn render_parts(parts: &[MessagePart]) -> Vec { subagent_ref: subagent_ref.clone(), subagent_status: *subagent_status, subagent_tail: subagent_tail.clone(), + preview_paths: preview_paths.clone(), }, other => other.clone(), }) @@ -5190,6 +5305,147 @@ mod tests { assert!(!journal.contains("secret")); } + #[tokio::test] + async fn tool_result_images_are_stored_inside_uploads() { + use base64::Engine as _; + let dir = tempfile::tempdir().unwrap(); + let uploads_dir = dir.path().join("uploads"); + std::fs::create_dir_all(&uploads_dir).unwrap(); + let sessions = SessionsEngine::new( + "host".into(), + Arc::new(RunJournal::open(dir.path().join("journals")).unwrap()), + Arc::new(HarnessRegistry::new()), + ); + let uploads = crate::uploads::Uploads::from_root(&uploads_dir); + sessions.set_generated_images(uploads.clone(), dir.path().join("generated")); + let png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + let payload = || AgentEvent::Subagent { + parent_tool_use_id: "child".into(), + event: Box::new(AgentEvent::ToolResult { + id: "tool-1".into(), + is_error: false, + output: None, + diff: None, + preview_paths: Vec::new(), + image_payloads: vec![zeron_proto::ToolResultImage { + name: "1.png".into(), + data_base64: png_b64.into(), + }], + }), + }; + let events = sessions + .inner + .prepare_generated_image("chat", payload(), &mut std::collections::HashSet::new()) + .await; + assert_eq!(events.len(), 1); + let AgentEvent::Subagent { + parent_tool_use_id, + event, + } = &events[0] + else { + panic!("lost owner"); + }; + assert_eq!(parent_tool_use_id, "child"); + let AgentEvent::ToolResult { + preview_paths, + image_payloads, + is_error, + .. + } = event.as_ref() + else { + panic!("expected a tool result"); + }; + assert!(!*is_error); + assert!(image_payloads.is_empty()); + assert_eq!(preview_paths.len(), 1); + assert!( + std::path::Path::new(&preview_paths[0]).starts_with(&uploads_dir), + "{}", + preview_paths[0] + ); + let chunk = uploads.read_chunk(&preview_paths[0], 0, &[]).unwrap(); + assert!(chunk.done); + assert_eq!( + base64::engine::general_purpose::STANDARD + .decode(chunk.data) + .unwrap(), + base64::engine::general_purpose::STANDARD + .decode(png_b64) + .unwrap() + ); + let again = sessions + .inner + .prepare_generated_image("chat", payload(), &mut std::collections::HashSet::new()) + .await; + let AgentEvent::Subagent { + event: again_event, .. + } = &again[0] + else { + panic!("lost owner"); + }; + let AgentEvent::ToolResult { + preview_paths: again_paths, + .. + } = again_event.as_ref() + else { + panic!("expected a tool result"); + }; + assert_eq!(again_paths, preview_paths); + sessions.inner.publish("chat", &events[0]); + let journal = + serde_json::to_string(&sessions.inner.journal.replay("chat", 0).unwrap()).unwrap(); + assert!(!journal.contains(png_b64)); + assert!(journal.contains(&preview_paths[0])); + + let bad = AgentEvent::ToolResult { + id: "tool-2".into(), + is_error: false, + output: None, + diff: None, + preview_paths: Vec::new(), + image_payloads: vec![zeron_proto::ToolResultImage { + name: "note.txt".into(), + data_base64: base64::engine::general_purpose::STANDARD + .encode(b"not an image") + .into(), + }], + }; + let rejected = sessions + .inner + .prepare_generated_image("chat", bad, &mut std::collections::HashSet::new()) + .await; + assert!(matches!( + &rejected[..], + [AgentEvent::ToolResult { + preview_paths, + image_payloads, + is_error: false, + .. + }] if preview_paths.is_empty() && image_payloads.is_empty() + )); + + let bare = SessionsEngine::new( + "host".into(), + Arc::new(RunJournal::open(dir.path().join("journals-bare")).unwrap()), + Arc::new(HarnessRegistry::new()), + ); + let dropped = bare + .inner + .prepare_generated_image("chat", payload(), &mut std::collections::HashSet::new()) + .await; + let AgentEvent::Subagent { event, .. } = &dropped[0] else { + panic!("lost owner"); + }; + assert!(matches!( + event.as_ref(), + AgentEvent::ToolResult { + preview_paths, + image_payloads, + .. + } if preview_paths.is_empty() && image_payloads.is_empty() + )); + } + /// The backwards journal scan must answer exactly what the old full forward /// replay did: newest non-empty id, cwd from the governing `SessionStarted`. #[tokio::test] diff --git a/crates/engine/src/uploads.rs b/crates/engine/src/uploads.rs index 63e7e73bb..a3937c636 100644 --- a/crates/engine/src/uploads.rs +++ b/crates/engine/src/uploads.rs @@ -1532,4 +1532,31 @@ mod generated_image_tests { std::fs::write(&source, b"changed").unwrap(); std::fs::rename(&parent, root.path().join("moved")).unwrap(); } + + #[test] + fn shown_image_is_sniffed_idempotent_and_readable_from_uploads() { + let store = tempfile::tempdir().unwrap(); + let uploads = Uploads::from_root(store.path()); + let png = BASE64 + .decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==") + .unwrap(); + let first = uploads.store_shown_image("same-id", "1.png", &png).unwrap(); + let again = uploads.store_shown_image("same-id", "1.png", &png).unwrap(); + assert_eq!(first, again); + assert_eq!(first.mime_type, "image/png"); + assert!(first.path.ends_with("-image.png")); + let chunk = uploads.read_chunk(&first.path, 0, &[]).unwrap(); + assert!(chunk.done); + assert_eq!(BASE64.decode(chunk.data).unwrap(), png); + assert!( + uploads + .store_shown_image("text", "x.bin", b"hello") + .is_err() + ); + assert!( + uploads + .store_shown_image("bad-png", "x.png", b"\x89PNG\r\n\x1a\nnot a png") + .is_err() + ); + } } diff --git a/crates/engine/tests/e2e.rs b/crates/engine/tests/e2e.rs index a7dfa841d..32a09189a 100644 --- a/crates/engine/tests/e2e.rs +++ b/crates/engine/tests/e2e.rs @@ -78,6 +78,8 @@ fn mock_script() -> Vec { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, done(DoneStatus::Completed), ] @@ -1334,6 +1336,7 @@ fn spawn_chip(id: &str, status: SubagentStatus) -> MessagePart { subagent_ref: Some(format!("{CHAT}--sub--{id}")), subagent_status: Some(status), subagent_tail: None, + preview_paths: Vec::new(), } } @@ -2873,6 +2876,8 @@ async fn stale_tool_echo_after_steer_boundary_does_not_split_text() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, AgentEvent::TextDelta { text: "two".into() }, done(DoneStatus::Completed), @@ -3276,6 +3281,8 @@ async fn generated_image_is_materialized_before_publication_and_survives_reopen( is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, image.clone(), image.clone(), diff --git a/crates/engine/tests/side_chats.rs b/crates/engine/tests/side_chats.rs index 67df50b36..2e531312e 100644 --- a/crates/engine/tests/side_chats.rs +++ b/crates/engine/tests/side_chats.rs @@ -433,6 +433,7 @@ async fn a_fork_settles_the_running_subagent_chips_it_copies() { subagent_ref: Some(format!("main--sub--{id}")), subagent_status: Some(status), subagent_tail: None, + preview_paths: Vec::new(), }; core.doc_host .open("main") diff --git a/crates/engine/tests/subagent_idle_reap.rs b/crates/engine/tests/subagent_idle_reap.rs index 35ca534d9..d4887b4b6 100644 --- a/crates/engine/tests/subagent_idle_reap.rs +++ b/crates/engine/tests/subagent_idle_reap.rs @@ -224,6 +224,8 @@ async fn park_after_spawn() -> Parked { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }) .unwrap(); feed.send(done(DoneStatus::Completed)).unwrap(); diff --git a/crates/engine/tests/turn_quiesce.rs b/crates/engine/tests/turn_quiesce.rs index 6e8dc63ef..b47b73166 100644 --- a/crates/engine/tests/turn_quiesce.rs +++ b/crates/engine/tests/turn_quiesce.rs @@ -439,6 +439,8 @@ async fn open_tool_call_never_quiesces() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }) .unwrap(); rig.feed.send(text("Build done.")).unwrap(); @@ -479,6 +481,8 @@ async fn stale_tool_echo_stays_parked() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }) .unwrap(); rig.feed diff --git a/crates/harness/src/acp/normalize.rs b/crates/harness/src/acp/normalize.rs index bb1e02122..7b7dac074 100644 --- a/crates/harness/src/acp/normalize.rs +++ b/crates/harness/src/acp/normalize.rs @@ -422,6 +422,8 @@ pub(crate) fn map_update(update: &Value) -> Vec { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ] } @@ -460,6 +462,8 @@ fn resolved_result(update: &Value, id: String) -> Option { is_error, output: tool_output(update), diff: tool_diff(update), + preview_paths: Vec::new(), + image_payloads: Vec::new(), }) } @@ -560,6 +564,8 @@ mod tests { is_error: false, output: Some("total 0".into()), diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ] ); @@ -603,6 +609,8 @@ mod tests { old_text: Some("fn old() {}".into()), new_text: "fn new() {}".into(), }), + preview_paths: Vec::new(), + image_payloads: Vec::new(), } ); } @@ -641,6 +649,8 @@ mod tests { is_error: true, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }] ); } diff --git a/crates/harness/src/acp/subagent.rs b/crates/harness/src/acp/subagent.rs index 4f80e7e58..8d8f4bd6b 100644 --- a/crates/harness/src/acp/subagent.rs +++ b/crates/harness/src/acp/subagent.rs @@ -402,6 +402,8 @@ fn entry_events(entry: &Value) -> Vec { is_error: false, output, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }); } } diff --git a/crates/harness/src/acp/system_message.rs b/crates/harness/src/acp/system_message.rs index 6ea15da01..96d6e90b2 100644 --- a/crates/harness/src/acp/system_message.rs +++ b/crates/harness/src/acp/system_message.rs @@ -266,6 +266,8 @@ mod tests { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ]; for reasoning in [false, true] { diff --git a/crates/harness/src/claude/normalize.rs b/crates/harness/src/claude/normalize.rs index acb29e712..d7d24b5be 100644 --- a/crates/harness/src/claude/normalize.rs +++ b/crates/harness/src/claude/normalize.rs @@ -262,7 +262,11 @@ fn tag(parent: &str, event: AgentEvent) -> AgentEvent { /// arrive only as full blocks on its tagged assistant frames — emit them, in /// block order with the tool calls, or subagent transcripts are /// tool-chips-only. -fn subagent_message(parent: &str, f: &MessageFrame) -> Vec { +fn subagent_message( + paths: &mut std::collections::HashMap, + parent: &str, + f: &MessageFrame, +) -> Vec { let mut out: Vec = f .message .blocks() @@ -279,13 +283,17 @@ fn subagent_message(parent: &str, f: &MessageFrame) -> Vec { text: format!("{}\n\n", b.thinking.trim_end()), }, )), - "tool_use" => Some(tag( - parent, - AgentEvent::ToolCall { - id: b.id.clone(), - call: decode_tool_use(&b.name, &b.input), - }, - )), + "tool_use" => { + let call = decode_tool_use(&b.name, &b.input); + remember_tool_file(paths, &b.id, &call); + Some(tag( + parent, + AgentEvent::ToolCall { + id: b.id.clone(), + call, + }, + )) + } _ => None, }) .collect(); @@ -300,23 +308,36 @@ fn subagent_message(parent: &str, f: &MessageFrame) -> Vec { out } -/// A subagent's tool results, attributed like its calls. -fn subagent_results(parent: &str, f: &MessageFrame) -> Vec { - f.message - .blocks() - .filter(|b: &ContentBlock| b.kind == "tool_result") - .map(|b| { - tag( - parent, - AgentEvent::ToolResult { - id: b.tool_use_id.clone(), - is_error: b.is_error.unwrap_or(false), - output: None, - diff: None, - }, - ) - }) - .collect() +fn remember_tool_file( + paths: &mut std::collections::HashMap, + id: &str, + call: &ToolCall, +) { + if id.is_empty() { + return; + } + let path = match call { + ToolCall::ReadFile { path } + | ToolCall::WriteFile { path, .. } + | ToolCall::EditFile { path, .. } => path.as_str(), + _ => return, + }; + let name = file_basename(path); + if name.is_empty() { + return; + } + paths.insert(id.to_owned(), name.to_owned()); +} + +fn empty_tool_result(id: impl Into, is_error: bool) -> AgentEvent { + AgentEvent::ToolResult { + id: id.into(), + is_error, + output: None, + diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), + } } /// An id the CLI minted, safe in a path. @@ -339,9 +360,128 @@ fn path_safe(id: &str) -> bool { /// /// Prefix-matched: the CLI ships at least two spellings of the marker /// (`…by user]` and `…by user for tool use]`). +/// +/// Also the CLI's image note, which rides a user frame after a tool result +/// that already carries the pixels: `[Image: source: ]` and +/// `[Image: original WxH, displayed at WxH. …]`. On a tagged frame it must +/// not read as a steer. fn is_synthetic_user_text(text: &str) -> bool { - let text = text.trim_start(); - text.starts_with("") || text.starts_with("[Request interrupted") + let text = text.trim(); + if text.starts_with("") || text.starts_with("[Request interrupted") { + return true; + } + let Some(rest) = text.strip_prefix("[Image: ") else { + return false; + }; + let Some(rest) = rest.strip_suffix(']') else { + return false; + }; + let rest = rest.trim_start(); + rest.starts_with("source: ") || rest.starts_with("original ") +} + +/// Images kept from one tool result. Matches the chip preview cap. +const MAX_TOOL_RESULT_IMAGES: usize = 4; + +fn is_base64_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'=') +} + +/// Copy `data` when it looks like base64 within the image size cap. +/// Whitespace is removed. Anything else is dropped before it is stored. +fn compact_base64(data: &str) -> Option { + let limit = zeron_proto::IMAGE_MAX_BYTES.div_ceil(3) * 4; + let mut count = 0usize; + for byte in data.bytes() { + if byte.is_ascii_whitespace() { + continue; + } + if !is_base64_byte(byte) { + return None; + } + count += 1; + if count > limit { + return None; + } + } + if count == 0 { + return None; + } + if count == data.len() { + return Some(data.to_owned()); + } + Some( + data.bytes() + .filter(|byte| !byte.is_ascii_whitespace()) + .map(char::from) + .collect(), + ) +} + +fn push_tool_image(out: &mut Vec, name: &str, data: &str) { + if out.len() >= MAX_TOOL_RESULT_IMAGES { + return; + } + let Some(data_base64) = compact_base64(data) else { + return; + }; + out.push(zeron_proto::ToolResultImage { + name: name.to_owned(), + data_base64, + }); +} + +/// Image blocks inside a tool result's `content` array. +fn images_in_content(content: Option<&Value>, name: &str) -> Vec { + let mut out = Vec::new(); + let Some(blocks) = content.and_then(Value::as_array) else { + return out; + }; + for block in blocks { + if out.len() >= MAX_TOOL_RESULT_IMAGES { + break; + } + if block.get("type").and_then(Value::as_str) != Some("image") { + continue; + } + if let Some(kind) = block.pointer("/source/type").and_then(Value::as_str) + && kind != "base64" + { + continue; + } + let Some(data) = block + .pointer("/source/data") + .and_then(Value::as_str) + .or_else(|| block.get("data").and_then(Value::as_str)) + else { + continue; + }; + push_tool_image(&mut out, name, data); + } + out +} + +/// `toolUseResult` image, used only when the content blocks had none. +fn image_from_tool_use_result( + value: Option<&Value>, + name: &str, +) -> Option { + let value = value?; + if value.get("type").and_then(Value::as_str) != Some("image") { + return None; + } + let data = value.pointer("/file/base64").and_then(Value::as_str)?; + let data_base64 = compact_base64(data)?; + Some(zeron_proto::ToolResultImage { + name: name.to_owned(), + data_base64, + }) +} + +fn file_basename(path: &str) -> &str { + path.rsplit(['/', '\\']) + .find(|part| !part.is_empty()) + .unwrap_or("image.png") } /// What `--resume ` would find in Claude's native history. @@ -418,6 +558,10 @@ pub(crate) struct Normalizer { /// which then opened as an empty, never-created subagent doc (user /// report 2026-08-20). agent_spawn_tools: std::collections::HashSet, + /// tool_use id → file basename for Read/Write/Edit, so an image result + /// can be labeled with the file the tool named. Removed when the result + /// arrives. + tool_paths: std::collections::HashMap, /// The typed fork of the turn under way (a turn runs one at most). Its /// replay feeds its chip only until the turn ends: a `forked-command-*` /// id after that stays the CLI's own, which no chip carries. @@ -447,6 +591,7 @@ impl Normalizer { agent_tool_spawns: std::collections::HashMap::new(), skill_calls: std::collections::HashMap::new(), agent_spawn_tools: std::collections::HashSet::new(), + tool_paths: std::collections::HashMap::new(), typed_fork: None, fork_transcripts: true, assistant_message_id: new_message_id(), @@ -456,6 +601,61 @@ impl Normalizer { } } + /// Tool results on a user frame. Image bytes ride the result's content + /// blocks (or `toolUseResult` when those blocks have none) and stay off + /// the journal: only `image_payloads` carries them, and the engine strips + /// that before publish. + fn tool_result_events( + &mut self, + frame: &MessageFrame, + clear_inline_skills: bool, + ) -> Vec { + let Some(blocks) = frame.message.content.as_array() else { + return Vec::new(); + }; + let mut pending = Vec::new(); + for block in blocks { + // Parsed for the result's id and error flag. Nested image bytes + // are not on that struct, so those still come from the raw value. + let Ok(parsed) = serde_json::from_value::(block.clone()) else { + continue; + }; + if parsed.kind != "tool_result" { + continue; + } + let id = parsed.tool_use_id; + if clear_inline_skills { + self.skill_calls.remove(&id); + } + let is_error = parsed.is_error.unwrap_or(false); + let name = self + .tool_paths + .remove(&id) + .unwrap_or_else(|| "image.png".into()); + let images = images_in_content(block.get("content"), &name); + pending.push((id, is_error, name, images)); + } + if pending.len() == 1 && pending[0].3.is_empty() { + let name = pending[0].2.clone(); + if let Some(image) = image_from_tool_use_result(frame.tool_use_result.as_ref(), &name) { + pending[0].3.push(image); + } + } + pending + .into_iter() + .map( + |(id, is_error, _name, image_payloads)| AgentEvent::ToolResult { + id, + is_error, + output: None, + diff: None, + preview_paths: Vec::new(), + image_payloads, + }, + ) + .collect() + } + /// The CLI reported its session (`system:init`). pub fn started(&self) -> bool { self.saw_init @@ -648,12 +848,7 @@ impl Normalizer { session_id: None, }, ), - AgentEvent::ToolResult { - id: fork.chip, - is_error: status == DoneStatus::Errored, - output: None, - diff: None, - }, + empty_tool_result(&fork.chip, status == DoneStatus::Errored), ] } @@ -683,10 +878,11 @@ impl Normalizer { /// records) are not conversation. Once the replay has begun, it brings /// the rest. pub(crate) fn fork_record(&mut self, record: Value) -> Vec { - let Some(fork) = self + let Some(chip) = self .typed_fork - .as_mut() + .as_ref() .filter(|fork| fork.replayed.is_none()) + .map(|fork| fork.chip.clone()) else { return Vec::new(); }; @@ -699,12 +895,18 @@ impl Normalizer { return Vec::new(); }; let events = if assistant { - subagent_message(&fork.chip, &f) + subagent_message(&mut self.tool_paths, &chip, &f) } else { - subagent_results(&fork.chip, &f) + // Disk replay shows tool results only, the same as before. Image + // bytes on those results are kept; a scale note is not a message. + self.tool_result_events(&f, false) + .into_iter() + .map(|event| tag(&chip, event)) + .collect() }; if !events.is_empty() && let Some(uuid) = f.uuid + && let Some(fork) = self.typed_fork.as_mut() { fork.seen.insert(uuid); } @@ -992,7 +1194,7 @@ impl Normalizer { if self.fork_repeat(parent, f.uuid.as_deref()) { return Vec::new(); } - return subagent_message(parent, &f); + return subagent_message(&mut self.tool_paths, parent, &f); } // Record spawn tool ids up front: `task_notification` keys on // them; a task may finish before a `task_started` arrives. A @@ -1017,9 +1219,11 @@ impl Normalizer { .blocks() .filter(|b: &ContentBlock| b.kind == "tool_use") .flat_map(|b| { + let decoded = decode_tool_use(&b.name, &b.input); + remember_tool_file(&mut self.tool_paths, &b.id, &decoded); let call = AgentEvent::ToolCall { id: b.id.clone(), - call: decode_tool_use(&b.name, &b.input), + call: decoded, }; // A spawn's `prompt` is the subagent's opening user // message — the wire never echoes it on the child @@ -1112,7 +1316,16 @@ impl Normalizer { } // A subagent's tool results echo on the main channel too; // they belong to its transcript, attributed like its calls. - let mut out = subagent_results(parent, &f); + let mut out = self + .tool_result_events(&f, false) + .into_iter() + .map(|event| tag(parent, event)) + .collect::>(); + // A meta frame's text is bookkeeping (the image scale note + // is one). It must not read as a steer. + if f.is_meta { + return out; + } // A tagged user frame's TEXT blocks are the parent // steering its subagent (SendMessage-style follow-ups — // tool results ride their own blocks, filtered above). @@ -1143,21 +1356,10 @@ impl Normalizer { ); return out; } - f.message - .blocks() - .filter(|b: &ContentBlock| b.kind == "tool_result") - .map(|b| { - // A skill whose result came back without a task - // start ran inline: it never forks now. - self.skill_calls.remove(&b.tool_use_id); - AgentEvent::ToolResult { - id: b.tool_use_id.clone(), - is_error: b.is_error.unwrap_or(false), - output: None, - diff: None, - } - }) - .collect() + // A skill whose result came back without a task start ran + // inline: it never forks now. `tool_result_events` drops that + // pending skill when asked. + self.tool_result_events(&f, true) } // A claude.ai plan window was hit. A hard `rejected` blocks the @@ -1517,12 +1719,7 @@ mod tests { ev, vec![AgentEvent::Subagent { parent_tool_use_id: "toolu_sub".into(), - event: Box::new(AgentEvent::ToolResult { - id: "t1".into(), - is_error: false, - output: None, - diff: None, - }), + event: Box::new(empty_tool_result("t1", false)), }] ); } @@ -1592,6 +1789,139 @@ mod tests { ); } + /// A tool that returns an image puts the pixels on the result. The + /// following note about the displayed size is bookkeeping, not a steer, + /// on the main feed and on a subagent feed. + #[test] + fn image_tool_results_keep_bytes_and_the_scale_note_is_not_a_steer() { + const PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + let scale = "[Image: original 1884x2066, displayed at 1824x2000. Multiply coordinates by 1.03 to map to original image.]"; + let source = "[Image: source: /tmp/shots/1.png]"; + let combined = "[Image: source: /tmp/shots/1.png, original 10x10, displayed at 10x10.]"; + + let mut norm = Normalizer::new(); + let read = crate::claude::wire::parse_frame( + r#"{"type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu_read","name":"Read","input":{"file_path":"/tmp/shots/1.png"}}]}}"#, + ) + .expect("parses"); + norm.normalize(read, false); + let result = crate::claude::wire::parse_frame(&format!( + r#"{{"type":"user","message":{{"content":[{{"type":"tool_result","tool_use_id":"toolu_read","content":[{{"type":"image","source":{{"type":"base64","media_type":"image/png","data":"{PNG}"}}}}]}}]}}}}"#, + PNG = PNG + )) + .expect("parses"); + let ev = norm.normalize(result, false); + let AgentEvent::ToolResult { + id, + image_payloads, + preview_paths, + .. + } = &ev[0] + else { + panic!("expected a tool result, got {ev:?}"); + }; + assert_eq!(id, "toolu_read"); + assert!(preview_paths.is_empty()); + assert_eq!(image_payloads.len(), 1); + assert_eq!(image_payloads[0].name, "1.png"); + assert_eq!(image_payloads[0].data_base64, PNG); + let journaled = serde_json::to_string(&ev[0]).unwrap(); + assert!( + !journaled.contains(PNG), + "image bytes must not be serialized" + ); + assert!(!journaled.contains("imagePayloads")); + + // The same bytes on a subagent feed stay on the tool result. + let mut norm = Normalizer::new(); + let read = crate::claude::wire::parse_frame( + r#"{"type":"assistant","parent_tool_use_id":"toolu_sub","message":{"content":[{"type":"tool_use","id":"toolu_child","name":"Read","input":{"file_path":"/tmp/shots/00.png"}}]}}"#, + ) + .expect("parses"); + norm.normalize(read, false); + let result = crate::claude::wire::parse_frame(&format!( + r#"{{"type":"user","parent_tool_use_id":"toolu_sub","message":{{"content":[{{"type":"tool_result","tool_use_id":"toolu_child","content":[{{"type":"image","source":{{"type":"base64","data":"{PNG}"}}}}]}}]}}}}"#, + PNG = PNG + )) + .expect("parses"); + let ev = norm.normalize(result, false); + assert!(matches!( + &ev[..], + [AgentEvent::Subagent { parent_tool_use_id, event }] + if parent_tool_use_id == "toolu_sub" + && matches!(event.as_ref(), AgentEvent::ToolResult { image_payloads, .. } if image_payloads.len() == 1 && image_payloads[0].name == "00.png") + )); + + for note in [scale, source, combined] { + let frame = format!( + r#"{{"type":"user","parent_tool_use_id":"toolu_sub","message":{{"content":[{{"type":"text","text":{text}}}]}}}}"#, + text = serde_json::to_string(note).unwrap() + ); + assert!( + !normalize_one(&frame) + .iter() + .any(|event| matches!(event, AgentEvent::Subagent { .. })), + "{note} leaked as a steer" + ); + let meta = format!( + r#"{{"type":"user","isMeta":true,"parent_tool_use_id":"toolu_sub","message":{{"content":{text}}}}}"#, + text = serde_json::to_string(note).unwrap() + ); + assert!( + normalize_one(&meta).is_empty(), + "string meta note leaked: {note}" + ); + } + // A meta frame cannot resurrect a settled chip, even with ordinary wording. + let meta_steer = r#"{"type":"user","isMeta":true,"parent_tool_use_id":"toolu_sub","message":{"content":[{"type":"text","text":"Keep going."}]}}"#; + assert!(normalize_one(meta_steer).is_empty()); + // A real steer on the same frame shape still arrives. + let real = r#"{"type":"user","parent_tool_use_id":"toolu_sub","message":{"content":[{"type":"text","text":"Keep going."}]}}"#; + assert!(normalize_one(real).iter().any(|event| matches!( + event, + AgentEvent::Subagent { event: inner, .. } + if matches!(inner.as_ref(), AgentEvent::UserMessage { text } if text == "Keep going.") + ))); + + // Fallback when the content block has no image and the frame has one result. + let fallback = crate::claude::wire::parse_frame(&format!( + r#"{{"type":"user","toolUseResult":{{"type":"image","file":{{"base64":"{PNG}","type":"image/png"}}}},"message":{{"content":[{{"type":"tool_result","tool_use_id":"toolu_fb","content":"ok"}}]}}}}"#, + PNG = PNG + )) + .expect("parses"); + let ev = Normalizer::new().normalize(fallback, false); + assert!(matches!( + &ev[..], + [AgentEvent::ToolResult { image_payloads, .. }] if image_payloads.len() == 1 && image_payloads[0].name == "image.png" + )); + // A result that already has the image does not take the fallback as a second copy. + let both = crate::claude::wire::parse_frame(&format!( + r#"{{"type":"user","toolUseResult":{{"type":"image","file":{{"base64":"{PNG}"}}}},"message":{{"content":[{{"type":"tool_result","tool_use_id":"toolu_both","content":[{{"type":"image","source":{{"type":"base64","data":"{PNG}"}}}}]}}]}}}}"#, + PNG = PNG + )) + .expect("parses"); + let ev = Normalizer::new().normalize(both, false); + assert!(matches!( + &ev[..], + [AgentEvent::ToolResult { image_payloads, .. }] if image_payloads.len() == 1 + )); + + // Not an image, or too large to be one: no payload. + let junk = crate::claude::wire::parse_frame( + r#"{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"toolu_junk","content":[{"type":"image","source":{"type":"base64","data":"!!!!"}}]}]}}"#, + ) + .expect("parses"); + let ev = Normalizer::new().normalize(junk, false); + assert!(matches!( + &ev[..], + [AgentEvent::ToolResult { image_payloads, .. }] if image_payloads.is_empty() + )); + let mut huge = "A".repeat(zeron_proto::IMAGE_MAX_BYTES.div_ceil(3) * 4 + 1); + assert!(compact_base64(&huge).is_none()); + huge.clear(); + assert_eq!(compact_base64("aGVs\nbG8=").as_deref(), Some("aGVsbG8=")); + } + #[test] fn send_message_steers_rekey_onto_the_spawn_feed() { // Live 2.1.228: the steer NEVER echoes on the child feed; the only @@ -2021,12 +2351,7 @@ mod tests { session_id: None, }, ), - AgentEvent::ToolResult { - id: TYPED_CHIP.into(), - is_error: status == DoneStatus::Errored, - output: None, - diff: None, - }, + empty_tool_result(TYPED_CHIP, status == DoneStatus::Errored), ] } diff --git a/crates/harness/src/claude/wire.rs b/crates/harness/src/claude/wire.rs index c2e5ffb97..e25c52dca 100644 --- a/crates/harness/src/claude/wire.rs +++ b/crates/harness/src/claude/wire.rs @@ -173,6 +173,13 @@ pub(crate) struct MessageFrame { /// Terse assistant-level error code (`rate_limit`, `billing_error`, …). #[serde(default)] pub error: Option, + /// Bookkeeping frame, not a person typing. Image scale notes ride this. + #[serde(default, rename = "isMeta")] + pub is_meta: bool, + /// Structured tool outcome. An image result repeats its bytes here when + /// the content blocks do not. + #[serde(default, rename = "toolUseResult")] + pub tool_use_result: Option, } #[derive(Debug, Default, Deserialize)] diff --git a/crates/harness/src/codex/normalize.rs b/crates/harness/src/codex/normalize.rs index cdc93855a..cfb65efda 100644 --- a/crates/harness/src/codex/normalize.rs +++ b/crates/harness/src/codex/normalize.rs @@ -329,6 +329,8 @@ fn tool_lifecycle(phase: Phase, id: String, call: ToolCall, is_error: bool) -> V is_error, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ], } @@ -458,6 +460,8 @@ pub(crate) fn map_item(phase: Phase, item: &Value) -> Vec { is_error: error.is_some(), output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ]; if let Some(message) = error { @@ -495,6 +499,8 @@ pub(crate) fn map_item(phase: Phase, item: &Value) -> Vec { is_error: status == "failed" || exit_code != 0, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }] } }, @@ -550,6 +556,8 @@ pub(crate) fn map_item(phase: Phase, item: &Value) -> Vec { is_error: status == "failed", output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }], }, "webSearch" | "web_search" => tool_lifecycle( @@ -996,6 +1004,8 @@ mod tests { is_error: true, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }] ); } @@ -1037,6 +1047,8 @@ mod tests { is_error: true, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ] ); @@ -1081,6 +1093,8 @@ mod tests { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ] ); @@ -1115,6 +1129,8 @@ mod tests { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ] ); diff --git a/crates/harness/src/cursor/mod.rs b/crates/harness/src/cursor/mod.rs index 01c5d4ed1..3507ac386 100644 --- a/crates/harness/src/cursor/mod.rs +++ b/crates/harness/src/cursor/mod.rs @@ -1102,6 +1102,8 @@ fn map_shim_frame(frame: &Value, interrupted: bool) -> Vec { is_error, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }), ]; // A finished task IS the subagent finishing: the SDK has diff --git a/crates/harness/src/mock.rs b/crates/harness/src/mock.rs index ab6d74402..a1ba91ae7 100644 --- a/crates/harness/src/mock.rs +++ b/crates/harness/src/mock.rs @@ -106,6 +106,8 @@ fn fanout_settle(i: usize) -> Vec { is_error: failed, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, fanout_tag( i, @@ -319,6 +321,8 @@ impl Harness for MockHarness { is_error: false, output: Some("test result: ok. 128 passed; 0 failed".into()), diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }); send("All 128 tests pass."); } else { @@ -528,6 +532,8 @@ impl Harness for MockHarness { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ] }) @@ -566,6 +572,8 @@ impl Harness for MockHarness { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }; let done = AgentEvent::Done { status: DoneStatus::Completed, @@ -731,6 +739,8 @@ impl Harness for MockHarness { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ] }) @@ -787,6 +797,8 @@ impl Harness for MockHarness { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }); events.push(AgentEvent::TextDelta { text: format!("Update {} written.\n\n", active + 1), diff --git a/crates/harness/src/muse/normalize.rs b/crates/harness/src/muse/normalize.rs index 93861aff2..14fea2aa3 100644 --- a/crates/harness/src/muse/normalize.rs +++ b/crates/harness/src/muse/normalize.rs @@ -159,6 +159,8 @@ impl Normalizer { || item["exitCode"].as_i64().is_some_and(|c| c != 0), output, diff, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }); } events @@ -201,6 +203,8 @@ impl Normalizer { is_error: status != "completed", output, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }); } } @@ -233,6 +237,8 @@ impl Normalizer { .or_else(|| item["failureReason"].as_str()) .map(|text| cap(text, 16384)), diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }); } } diff --git a/crates/harness/src/opencode/mod.rs b/crates/harness/src/opencode/mod.rs index 75237b410..0a586dbd3 100644 --- a/crates/harness/src/opencode/mod.rs +++ b/crates/harness/src/opencode/mod.rs @@ -4507,6 +4507,8 @@ fn part_snapshot_events( is_error: status == "error", output, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }); } events diff --git a/crates/harness/src/pi/normalize.rs b/crates/harness/src/pi/normalize.rs index 23ff8bb21..8e75ce62e 100644 --- a/crates/harness/src/pi/normalize.rs +++ b/crates/harness/src/pi/normalize.rs @@ -144,6 +144,8 @@ impl Normalizer { is_error: frame["isError"].as_bool().unwrap_or(false), output: output.map(|s| cap(&s, 16384)), diff, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }); } "compaction_end" => { diff --git a/crates/harness/tests/claude.rs b/crates/harness/tests/claude.rs index c94e9cdd8..0f6a3336b 100644 --- a/crates/harness/tests/claude.rs +++ b/crates/harness/tests/claude.rs @@ -169,6 +169,8 @@ async fn happy_path_normalizes_events_and_tags_subagents() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }), })); @@ -197,12 +199,16 @@ async fn happy_path_normalizes_events_and_tags_subagents() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), })); assert!(events.contains(&AgentEvent::ToolResult { id: "tool-2".into(), is_error: true, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), })); // Informational rate-limit frames stay quiet. diff --git a/crates/harness/tests/codex.rs b/crates/harness/tests/codex.rs index 4ce5482e7..b1933a198 100644 --- a/crates/harness/tests/codex.rs +++ b/crates/harness/tests/codex.rs @@ -239,6 +239,8 @@ async fn happy_path_maps_deltas_items_usage_and_done() { is_error: true, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), })); // fileChange (single add): WriteFile, refreshed at completion. @@ -261,6 +263,8 @@ async fn happy_path_maps_deltas_items_usage_and_done() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), })); // mcpToolCall with failed status. @@ -277,6 +281,8 @@ async fn happy_path_maps_deltas_items_usage_and_done() { is_error: true, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), })); // webSearch lifecycle. @@ -291,6 +297,8 @@ async fn happy_path_maps_deltas_items_usage_and_done() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), })); // Completion-only todoList still opens and closes the lifecycle. @@ -308,6 +316,8 @@ async fn happy_path_maps_deltas_items_usage_and_done() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), })); // Streamed agentMessage must not re-emit its completed text… @@ -1449,6 +1459,8 @@ async fn child_thread_routing_tags_and_never_settles_parent() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }), })); // The parent's steer (a userMessage item on the CHILD thread) arrives as @@ -1864,6 +1876,8 @@ async fn view_image_reads_as_a_file_read() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), }, ] ); diff --git a/crates/harness/tests/cursor.rs b/crates/harness/tests/cursor.rs index 4404d38e9..569d9c94b 100644 --- a/crates/harness/tests/cursor.rs +++ b/crates/harness/tests/cursor.rs @@ -305,6 +305,8 @@ async fn happy_path_maps_shim_frames_and_tags_subagents() { is_error: false, output: None, diff: None, + preview_paths: Vec::new(), + image_payloads: Vec::new(), })); // The task spawn is a parent chip; its interior arrives tagged, never bare. diff --git a/crates/mcp/src/transcript.rs b/crates/mcp/src/transcript.rs index 788cff80a..ca3ad5538 100644 --- a/crates/mcp/src/transcript.rs +++ b/crates/mcp/src/transcript.rs @@ -284,6 +284,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }, MessagePart::Text { id: "t2".into(), diff --git a/crates/mobile/src/layout/tests.rs b/crates/mobile/src/layout/tests.rs index 34df2ab80..8093d3199 100644 --- a/crates/mobile/src/layout/tests.rs +++ b/crates/mobile/src/layout/tests.rs @@ -161,6 +161,7 @@ fn messages_the_agent_sent_show_as_one_line_naming_the_recipient() { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }; let mut w = worker(390.0); w.input = TranscriptInput { @@ -680,6 +681,7 @@ fn running_subagent_shows_a_spinner_after_its_spawn_resolves() { subagent_ref: Some("sub-1".into()), subagent_status: Some(status), subagent_tail: None, + preview_paths: Vec::new(), }; let frame_for = |status: SubagentStatus| { let mut w = worker(390.0); diff --git a/crates/proto/src/agent.rs b/crates/proto/src/agent.rs index 99ed54fb6..5b8ea6c76 100644 --- a/crates/proto/src/agent.rs +++ b/crates/proto/src/agent.rs @@ -746,6 +746,14 @@ pub struct SlashCommand { pub input_hint: Option, } +/// One image embedded in a tool result, held only until the engine stores +/// a copy under the profile uploads directory. Never journaled. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ToolResultImage { + pub name: String, + pub data_base64: String, +} + /// A file modification carried inline on a tool result (ACP /// `ToolCallContent::Diff`). `old_text: None` means a new file. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -850,6 +858,13 @@ pub enum AgentEvent { /// Inline file diff for edit-shaped tools (ACP `Diff` content). #[serde(default, skip_serializing_if = "Option::is_none")] diff: Option, + /// Uploads paths of images that were embedded in this result. The + /// engine fills this before journaling; adapters leave it empty. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + preview_paths: Vec, + /// In-memory image bytes for the engine to store. Never serialized. + #[serde(skip)] + image_payloads: Vec, }, /// Latest context occupancy, independent of cumulative billing usage. /// Missing fields preserve the previous measurement; zero tokens is valid. diff --git a/crates/ui/examples/subagent-fold-fixture.rs b/crates/ui/examples/subagent-fold-fixture.rs index 6fcb122a8..dcaa4975a 100644 --- a/crates/ui/examples/subagent-fold-fixture.rs +++ b/crates/ui/examples/subagent-fold-fixture.rs @@ -39,6 +39,7 @@ fn agent(id: &str, description: &str) -> MessagePart { subagent_ref: Some(format!("chat--sub--{id}")), subagent_status: Some(SubagentStatus::Running), subagent_tail: None, + preview_paths: Vec::new(), } } diff --git a/crates/ui/src/shell/session_lanes.rs b/crates/ui/src/shell/session_lanes.rs index 8ead8ea66..887d24e1a 100644 --- a/crates/ui/src/shell/session_lanes.rs +++ b/crates/ui/src/shell/session_lanes.rs @@ -741,6 +741,7 @@ mod tests { subagent_ref: doc.map(str::to_owned), subagent_status: status, subagent_tail: None, + preview_paths: Vec::new(), } } diff --git a/crates/ui/src/state.rs b/crates/ui/src/state.rs index a1494a538..1d4c2cf22 100644 --- a/crates/ui/src/state.rs +++ b/crates/ui/src/state.rs @@ -2175,6 +2175,7 @@ impl AppState { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }], created_at: created_at + 1, device_id: "local".into(), diff --git a/crates/ui/src/todo_panel.rs b/crates/ui/src/todo_panel.rs index 838b9c05a..e43fd3f6e 100644 --- a/crates/ui/src/todo_panel.rs +++ b/crates/ui/src/todo_panel.rs @@ -732,6 +732,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), } } @@ -988,6 +989,7 @@ mod composer_tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }], created_at: 0, device_id: "d".into(), diff --git a/crates/ui/src/tool_images.rs b/crates/ui/src/tool_images.rs index ba6229f07..1281677f0 100644 --- a/crates/ui/src/tool_images.rs +++ b/crates/ui/src/tool_images.rs @@ -115,6 +115,24 @@ fn push_json_paths(out: &mut Vec, value: &serde_json::Value) { } } +/// Paths the chip previews. A tool result that embedded image bytes names the +/// uploads copies here; those replace paths the call printed, which may sit +/// outside the attachment jail. An empty list keeps the call and output paths. +pub(crate) fn chip_images( + call: &ToolCall, + output: Option<&str>, + preview_paths: &[String], +) -> Arc<[SharedString]> { + if !preview_paths.is_empty() { + let mut out = Vec::new(); + for path in preview_paths { + push_path(&mut out, path); + } + return out.into(); + } + image_paths(call, output) +} + /// The image files a tool call touched, as reported (possibly relative to the /// chat's working directory). Computed once per row build, never per paint. pub(crate) fn image_paths(call: &ToolCall, output: Option<&str>) -> Arc<[SharedString]> { @@ -657,6 +675,33 @@ mod tests { ); } + #[test] + fn stored_preview_paths_replace_the_named_file() { + let call = ToolCall::ReadFile { + path: "/tmp/shots/1.png".into(), + }; + let stored = chip_images( + &call, + Some("also /tmp/other.png"), + &["/uploads/abc-image.png".into()], + ); + assert_eq!( + stored + .iter() + .map(|path| path.to_string()) + .collect::>(), + vec!["/uploads/abc-image.png".to_string()] + ); + let named = chip_images(&call, Some("also /tmp/other.png"), &[]); + assert_eq!( + named + .iter() + .map(|path| path.to_string()) + .collect::>(), + vec!["/tmp/shots/1.png".to_string(), "/tmp/other.png".to_string()] + ); + } + #[test] fn relative_paths_join_the_owning_hosts_cwd() { assert_eq!(resolve_path("a/b.png", Some("/w")), "/w/a/b.png"); diff --git a/crates/ui/src/transcript.rs b/crates/ui/src/transcript.rs index 7712be529..b0f013126 100644 --- a/crates/ui/src/transcript.rs +++ b/crates/ui/src/transcript.rs @@ -1542,6 +1542,7 @@ pub fn rows_for_entry( subagent_ref, subagent_status, subagent_tail, + preview_paths, .. } => { // A message to another chat reads as one line naming the @@ -1594,7 +1595,7 @@ pub fn rows_for_entry( subagent_status: *subagent_status, subagent_tail: subagent_tail.clone().map(SharedString::from), kind: ToolItemKind::Call, - images: crate::tool_images::image_paths(call, output.as_deref()), + images: crate::tool_images::chip_images(call, output.as_deref(), preview_paths), }; if compact { // Compact mode keeps ONE group for the whole turn — @@ -13256,6 +13257,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), }; let entry = assistant( "a1", @@ -14449,6 +14451,7 @@ mod tests { subagent_ref: None, subagent_status: None, subagent_tail: None, + preview_paths: Vec::new(), } } @@ -14571,6 +14574,7 @@ mod tests { subagent_ref: Some(format!("chat--sub--{id}")), subagent_status: Some(SubagentStatus::Running), subagent_tail: None, + preview_paths: Vec::new(), } } diff --git a/docs/image-preview.md b/docs/image-preview.md index 00d1c9ec0..09a230755 100644 --- a/docs/image-preview.md +++ b/docs/image-preview.md @@ -23,7 +23,7 @@ Each Files image surface admits at most 64 MiB of retained image resources, incl ## Tool-call image previews -A tool chip whose call or output names an image file (PNG, JPEG, GIF, WebP, BMP, SVG, TIFF; at most four per chip) previews it when the chip is expanded. Relative paths resolve against the chat's cwd on the owning host. Bytes arrive through `ReadAttachmentChunk`, so its jail applies: the owning host's uploads directory (plus any read-only legacy uploads roots) and the working directory of every chat in the workspace on that host, not just the chat that ran the tool. A projectless chat's working directory is the home directory, so any image under home on that host is readable. Other paths show *Image unavailable*. Network and device paths (UNC `\\host\share`, `//host/share`, `\\?\UNC\…`, `\\.\…`) and drive-relative paths (`C:x.png`) are never previewed. The engine also jails each path lexically, resolving `.`/`..` without touching the filesystem, before it canonicalizes the path, so a path outside every root is refused without being opened. +A tool chip whose call or output names an image file (PNG, JPEG, GIF, WebP, BMP, SVG, TIFF; at most four per chip) previews it when the chip is expanded. Relative paths resolve against the chat's cwd on the owning host. Bytes arrive through `ReadAttachmentChunk`, so its jail applies: the owning host's uploads directory (plus any read-only legacy uploads roots) and the working directory of every chat in the workspace on that host, not just the chat that ran the tool. A projectless chat's working directory is the home directory, so any image under home on that host is readable. When the tool result itself embeds image bytes, the engine stores a copy in the profile uploads directory and the chip previews that copy, so a scratch path named by the call does not have to be readable. A note that only describes the image's displayed size is not shown as a subagent message. Paths with no embedded bytes, and bytes the store rejects, still show *Image unavailable*. Network and device paths (UNC `\\host\share`, `//host/share`, `\\?\UNC\…`, `\\.\…`) and drive-relative paths (`C:x.png`) are never previewed. The engine also jails each path lexically, resolving `.`/`..` without touching the filesystem, before it canonicalizes the path, so a path outside every root is refused without being opened. `tool_images.rs` decodes one static frame on the background executor within the shared raster limits (4096 pixels per side, 64 MiB decoder allocation), downsamples it to at most 1120×440 and hands BGRA pixels straight to GPUI as a `RenderImage` (no re-encode, no asset cache). Nothing is read or decoded for collapsed chips or hidden groups. Collapsing starts a 600 ms grace; then the CPU frame and the sprite-atlas tiles are released. Reopening inside the grace reuses the texture, so rapid toggling never re-reads or re-decodes. Previews that are expanded but unpainted for 2 s count against a 64 MiB budget; painted ones are never evicted. Chat switches and transcript disposal release everything. From d313d810ab0af8fc8cb3637e67189a8d5f197305 Mon Sep 17 00:00:00 2001 From: Daniel Katulevskiy Date: Tue, 3 Nov 2026 14:35:09 -0800 Subject: [PATCH 02/81] zgx: a tool's result pictures show in its opened row; Claude Code's image note is no message A Read of a picture whose result carried its pixels now opens to that picture, as Zeron's chip preview does: the engine keeps a copy in the profile's uploads (the tool part's previewPaths), and the row reads it where the transcript reads an agent's shown pictures. One alone or several on a tray, opening in the lightbox; the name is the file the call named. A tool that only named a path shows none. A user entry that is only Claude Code's note on such a picture ("[Image: original WxH, displayed at WxH. ...]", "[Image: source: ...]"), which subagent transcripts kept as a steer before the engine stopped folding it, no longer shows as a bubble. --- apps/zgx/src/chat.rs | 53 ++++++++++++++++++++++ apps/zgx/src/tool_details.rs | 55 +++++++++++++++++++++++ apps/zgx/ui/ds/activity.luau | 54 ++++++++++++++++++++-- apps/zgx/ui/ds/transcript.luau | 2 + apps/zgx/ui/tests/tool_pictures.test.luau | 37 +++++++++++++++ apps/zgx/ui/tool_pictures_fixture.luau | 49 ++++++++++++++++++++ 6 files changed, 246 insertions(+), 4 deletions(-) create mode 100644 apps/zgx/ui/tests/tool_pictures.test.luau create mode 100644 apps/zgx/ui/tool_pictures_fixture.luau diff --git a/apps/zgx/src/chat.rs b/apps/zgx/src/chat.rs index 4d9d6aee8..b0f6011f9 100644 --- a/apps/zgx/src/chat.rs +++ b/apps/zgx/src/chat.rs @@ -2379,6 +2379,34 @@ fn entry_text(entry: &Json) -> String { out } +/// Whether a user entry is only Claude Code's note on a picture a tool read +/// (`[Image: original WxH, displayed at WxH. …]`, `[Image: source: …]`), as +/// zeron-harness's `is_synthetic_user_text` reads it: nothing else in it. +fn image_note_only(entry: &Json) -> bool { + let mut notes = 0; + for p in entry["parts"].as_array().into_iter().flatten() { + match p["kind"].as_str() { + Some("text") => { + let text = p["text"].as_str().unwrap_or_default().trim(); + if text.is_empty() { + continue; + } + let note = text + .strip_prefix("[Image: ") + .and_then(|rest| rest.strip_suffix(']')) + .map(str::trim_start) + .is_some_and(|rest| rest.starts_with("source: ") || rest.starts_with("original ")); + if !note { + return false; + } + notes += 1; + } + _ => return false, + } + } + notes > 0 +} + /// An entry as ds/transcript's message; nil for one it doesn't show. pub(crate) fn message(entry: &Json, model: Option<&str>) -> Option { let id = entry["id"].as_str()?; @@ -2392,6 +2420,13 @@ pub(crate) fn message(entry: &Json, model: Option<&str>) -> Option { .filter_map(|p| p["text"].as_str()) .collect::>() .join("\n\n"); + // Claude Code's note on a picture a tool read (its size as the + // model saw it), which a subagent's transcript kept as a message + // before the engine learned it isn't one: not something anyone + // said, so no bubble. + if image_note_only(entry) { + return None; + } let (words, mut paths) = split_attachments(&text); // (An Appshot's context isn't what the person said: its picture // says which app instead.) @@ -3998,6 +4033,24 @@ mod tests { assert!(apply(&mut entries, &wrong).is_err()); } + #[test] + fn claude_codes_image_note_is_no_message() { + let user = |text: &str| { + json!({ "id": "u", "role": "user", "createdAt": 1_788_900_000_000_i64, + "parts": [{ "id": "t", "kind": "text", "text": text }] }) + }; + for note in [ + "[Image: original 1884x2066, displayed at 1824x2000. Multiply coordinates by 1.03 to map to original image.]", + "[Image: source: /home/me/scratch/test.png]", + " [Image: source: /a.png, original 10x10, displayed at 10x10.]\n", + ] { + assert!(message(&user(note), None).is_none(), "{note} showed"); + } + // Words of its own around it, or a note-like line that isn't one, show. + assert!(message(&user("Look: [Image: original 10x10, displayed at 10x10.]"), None).is_some()); + assert!(message(&user("[Image: the logo]"), None).is_some()); + } + #[test] fn a_prompts_attached_passages_show_as_chips_on_the_sent_message() { let text = "Why this one?\n\nSelected from this conversation:\n- GPT-5.5, Sep 8, 1:43 PM: Layout dominates.\n Paint is second.\n- You: the first ask\n\nAttached images (local files — open them to view):\n- /tmp/shot.png"; diff --git a/apps/zgx/src/tool_details.rs b/apps/zgx/src/tool_details.rs index c29fccde9..88cf67309 100644 --- a/apps/zgx/src/tool_details.rs +++ b/apps/zgx/src/tool_details.rs @@ -190,13 +190,68 @@ pub fn tool_fields(part: &Json) -> Vec<(&'static str, Value)> { { fields.push(("diff", diff)); } + if let Some(pictures) = pictures_value(part) { + fields.push(("pictures", pictures)); + } fields } +/// Pictures a tool row opens to, at most (the shipping chip's +/// `MAX_IMAGES_PER_TOOL`). +const MAX_PICTURES: usize = 4; + +/// The pictures a tool's result carried (`previewPaths`: the engine's copies +/// in the profile's uploads, on the chat's host), each `{ path, name }`: its +/// name the file the call named (a Read's), else the copy's own. Nil when it +/// carried none: a path a tool only named is never read from here. +fn pictures_value(part: &Json) -> Option { + let paths: Vec<&str> = part["previewPaths"] + .as_array()? + .iter() + .filter_map(Json::as_str) + .filter(|p| !p.trim().is_empty()) + .take(MAX_PICTURES) + .collect(); + if paths.is_empty() { + return None; + } + let named = part["call"]["path"].as_str().and_then(file_name); + let list: Vec = paths + .iter() + .map(|path| { + let name = named.or_else(|| file_name(path)).unwrap_or_default(); + Value::record([("path", Value::from(*path)), ("name", Value::from(name))]) + }) + .collect(); + Some(Value::List(list.into())) +} + +fn file_name(path: &str) -> Option<&str> { + path.rsplit(['/', '\\']).find(|s| !s.is_empty()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn a_result_s_stored_pictures_are_the_row_s_named_as_the_call_s_file() { + let part = json!({ + "kind": "tool", "id": "t1", "isError": false, + "call": { "kind": "readFile", "path": "/home/me/scratch/shot.png" }, + "previewPaths": ["/home/me/.zeron/uploads/ab12-image.png"], + }); + let fields = tool_fields(&part); + let (_, pictures) = fields.iter().find(|(k, _)| *k == "pictures").expect("pictures"); + let Value::List(list) = pictures else { panic!("not a list: {pictures:?}") }; + assert_eq!(list.len(), 1); + assert_eq!(list[0].field("path").map(crate::auth::text).as_deref(), Some("/home/me/.zeron/uploads/ab12-image.png")); + assert_eq!(list[0].field("name").map(crate::auth::text).as_deref(), Some("shot.png")); + // None stored (an older doc, a path only named): no pictures. + let bare = json!({ "kind": "tool", "id": "t2", "call": { "kind": "readFile", "path": "/x/a.png" } }); + assert!(!tool_fields(&bare).iter().any(|(k, _)| *k == "pictures")); + } + #[test] fn a_diff_is_hunks_with_three_lines_of_context_numbered_from_one() { let old = (1..=20).map(|i| format!("line {i}\n")).collect::(); diff --git a/apps/zgx/ui/ds/activity.luau b/apps/zgx/ui/ds/activity.luau index 8fc155835..46300ef36 100644 --- a/apps/zgx/ui/ds/activity.luau +++ b/apps/zgx/ui/ds/activity.luau @@ -26,6 +26,7 @@ local status = require("@self/status") local ToolDetail = require("@self/tool_detail") local Markdown = require("@self/markdown") local locale = require("@self/locale") +local ReplyPicture = require("@self/reply_picture") export type Kind = "exec" | "read" | "edit" | "write" | "search" | "glob" | "web" | "todo" | "agent" | "thought" -- One tool: what kind, what it did ("Read", "transcript.rs"), and for an @@ -57,6 +58,10 @@ export type Tool = { diff_ref: string?, stats: { ToolDetail.Stat }?, diff: ToolDetail.Diff?, + -- The pictures its result carried (a Read of a screenshot), as the + -- engine keeps them: copies on the chat's host, which the row opens to + -- (Zeron's chip previews, crates/ui tool_images.rs). + pictures: { { path: string, name: string? } }?, } export type Props = { @@ -76,6 +81,9 @@ export type Props = { -- A subagent's spawn row (its tool's `subagent`): what opens it (its -- lane). Without it, the row opens its call as any does. open_subagent: ((sub: any) -> ())?, + -- Where a tool's pictures are read from (ds/transcript's + -- `shown_pictures`: files on the chat's host). Without it, none show. + pictures: ReplyPicture.Pictures?, } local GLYPH: { [Kind]: Icon.Name } = { @@ -380,7 +388,29 @@ end -- (`below`: a tool with a branch follows, so the trunk runs on under this -- one; `running`: it is the one in flight, its glyph the working loader.) -local function row(now: () -> Tool, below: () -> boolean, running: () -> boolean, open_subagent: ((sub: any) -> ())?): Node +-- A tool's pictures under its opened row: one alone, or several on a tray, +-- in a lightbox group of their own. +local function tool_pictures(now: () -> Tool, shown: ReplyPicture.Pictures): Node + local group = `tool:{now().id}` + local function entries(): { ReplyPicture.Entry } + local out = {} + for _, it in now().pictures or {} do + table.insert(out, { src = it.path, alt = it.name or "" }) + end + return out + end + return ui.presence(function(): string + return if #(now().pictures or {}) > 1 then "run" else "one" + end, function(kind: string): Node + if kind == "run" then + return ReplyPicture.Run { pictures = shown, items = entries, group = group, set = entries } + end + local first = entries()[1] or { src = "", alt = "" } + return ReplyPicture.Single { pictures = shown, src = first.src, alt = first.alt, group = group, set = entries } + end, { w_full = true, min_w = 0 }) +end + +local function row(now: () -> Tool, below: () -> boolean, running: () -> boolean, open_subagent: ((sub: any) -> ())?, pictures: ReplyPicture.Pictures?): Node if now().kind == "thought" then return thought_row(now, below) end @@ -396,9 +426,14 @@ local function row(now: () -> Tool, below: () -> boolean, running: () -> boolean local function has_detail(): boolean return ToolDetail.has(now() :: any) end + -- The pictures its result carried, where they can be read. + local function has_pictures(): boolean + local list = now().pictures + return pictures ~= nil and list ~= nil and #list > 0 + end -- Open as the reader left it (opened or closed), else closed. local function showing(): boolean - if not (has_call() or has_detail()) then + if not (has_call() or has_detail() or has_pictures()) then return false end return CALLS_OPEN:get()[id] == true @@ -460,7 +495,7 @@ local function row(now: () -> Tool, below: () -> boolean, running: () -> boolean open_subagent(sub) return end - if has_call() or has_detail() then + if has_call() or has_detail() or has_pictures() then local all = table.clone(CALLS_OPEN:peek()) all[id] = not showing() CALLS_OPEN:set(all) @@ -547,6 +582,17 @@ local function row(now: () -> Tool, below: () -> boolean, running: () -> boolean ui.column { w_full = true, ui.show(has_call, call_block(now)), + if pictures + then ui.show( + has_pictures, + ui.div { + ml = ui.raw(GUTTER, "under the line's words, past the tree's gutter"), + mt = "$space.1", + mb = "$space.1", + tool_pictures(now, pictures), + } + ) + else nil, ui.show( has_detail, ui.div { @@ -726,7 +772,7 @@ return ui.component(function(props: Props): Node return now:get() end, below_of(t), function(): boolean return live() and last() - end, props.open_subagent) + end, props.open_subagent, props.pictures) end, { key = function(t: Tool): string return t.id diff --git a/apps/zgx/ui/ds/transcript.luau b/apps/zgx/ui/ds/transcript.luau index 71c9121a3..94642eec5 100644 --- a/apps/zgx/ui/ds/transcript.luau +++ b/apps/zgx/ui/ds/transcript.luau @@ -467,6 +467,7 @@ local function blocks_of(list: () -> { Block }, m: Message, now: () -> Message, return Activity { key = `{m.id}:{b.id}`, open_subagent = if view then view.open_subagent else nil, + pictures = if view then view.shown_pictures else nil, tools = function() return brow:get().tools or {} end, @@ -619,6 +620,7 @@ local function reply( Activity { key = m.id, open_subagent = if view then view.open_subagent else nil, + pictures = if view then view.shown_pictures else nil, tools = function() return now().tools or {} end, diff --git a/apps/zgx/ui/tests/tool_pictures.test.luau b/apps/zgx/ui/tests/tool_pictures.test.luau new file mode 100644 index 000000000..ce07001ad --- /dev/null +++ b/apps/zgx/ui/tests/tool_pictures.test.luau @@ -0,0 +1,37 @@ +-- A tool whose result carried a picture (a Read of a screenshot: the +-- engine's copy, `previewPaths`) opens to that picture, as Zeron's chip +-- previews it; a click opens it in the lightbox. A Read that kept none +-- opens to nothing of the kind (ui/tool_pictures_fixture.luau). + +local function work(): any + local app = mount("tool_pictures_fixture.luau") + app:resize(900, 900) + app:frames(10) + expect(app:error()).to_be_nil() + -- (Settled work folds to its line: open it.) + app:click('role:button "Read 2 files"') + app:frames(30) + return app +end + +test("a read whose result kept a picture opens to it, and it opens in the lightbox", function() + local app = work() + expect(app:has('"Open test.png"')).to_be(false) + app:click('role:button "Read test.png"') + app:frames(30) + expect(app:error()).to_be_nil() + expect(app:has('"Open test.png"')).to_be(true) + app:click('"Open test.png"') + app:frames(60) + local open = app:state("ds/lightbox.luau::open") + expect(open).never.to_be_nil() + expect(open.items[open.index].src).to_be("assets/composer/shell.png") +end) + +test("a read that kept no picture shows none", function() + local app = work() + app:click('role:button "Read old.png"') + app:frames(30) + expect(app:error()).to_be_nil() + expect(app:has('"Open old.png"')).to_be(false) +end) diff --git a/apps/zgx/ui/tool_pictures_fixture.luau b/apps/zgx/ui/tool_pictures_fixture.luau new file mode 100644 index 000000000..3dbfaff5f --- /dev/null +++ b/apps/zgx/ui/tool_pictures_fixture.luau @@ -0,0 +1,49 @@ +-- A reply's work (ds/activity) with a Read of a picture whose result the +-- engine kept a copy of (its `pictures`, the tool's `previewPaths`), and a +-- Read of another that kept none, for tests/tool_pictures.test.luau. The +-- pictures are read from where the transcript reads an agent's shown ones +-- (stubbed: each path is the picture, nothing read). +-- Here, beside app.luau, as a mounted script loads modules from its own +-- directory down. + +ui.theme.strict("warn") +local ds = require("@ds") +ds.setup() +local Activity = require("@ds/activity") +local Lightbox = require("@ds/lightbox") + +local pictures = { + resolve = function(src: string): any + return { state = "ready", src = src, path = src, here = true } + end, + fetch = function(_src: string) end, +} + +local TOOLS = { + { + id = "r1", + kind = "read", + verb = "Read", + target = "test.png", + pictures = { { path = "assets/composer/shell.png", name = "test.png" } }, + }, + { id = "r2", kind = "read", verb = "Read", target = "old.png" }, +} + +return ui.column { + w_full = true, + h_full = true, + p = "$space.4", + ui.scroll { + w_full = true, + h_full = true, + Activity { + key = "work-pictures", + pictures = pictures, + tools = function(): { any } + return TOOLS + end, + }, + }, + Lightbox.Host(), +} From 72ee31fb61880972dbbf3fd49b98cdba5d5315f6 Mon Sep 17 00:00:00 2001 From: Daniel Katulevskiy Date: Tue, 3 Nov 2026 15:04:12 -0800 Subject: [PATCH 03/81] zgx: a subagent lane's tool pictures are read from its parent chat's host A subagent's doc has no chat row, so its lane's session is missing and the pictures its tools kept had nowhere to be read from ("couldn't be shown"). They are files on the parent's host: read them as the parent chat's are. --- apps/zgx/ui/screens/chat/lane_tab.luau | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/zgx/ui/screens/chat/lane_tab.luau b/apps/zgx/ui/screens/chat/lane_tab.luau index fcefdd7f8..17fde37ce 100644 --- a/apps/zgx/ui/screens/chat/lane_tab.luau +++ b/apps/zgx/ui/screens/chat/lane_tab.luau @@ -416,10 +416,18 @@ M.Tab = ui.component(function(props: { tab: any }): Node local pictures = ReplyImages.for_session(lane_session) local sent_pictures = ReplyImages.for_attachments(function(): any local s = lane_session() - if not s then + if s then + return { id = s.id, device = s.device, cwd = s.cwd, here = sessions.now:get().here } + end + -- A subagent's lane has no chat of its own (its doc is its parent's + -- child): the pictures its tools kept are files on its parent's + -- host, read as the parent's are. + local parent = data.parent + local open = store.session:get() + if not parent or not open or open.id ~= parent or open.state == "missing" then return nil end - return { id = s.id, device = s.device, cwd = s.cwd, here = sessions.now:get().here } + return { id = open.id, device = open.device, cwd = open.cwd, here = sessions.now:get().here } end) -- The session whose panel it's in (the page's): where a file it links -- opens, and a subagent it spawned. From 5455ba4d58e0a8c4b66e3f4712cddf6c4a946093 Mon Sep 17 00:00:00 2001 From: Daniel Katulevskiy Date: Tue, 3 Nov 2026 17:46:33 -0800 Subject: [PATCH 04/81] zgx: a tool's result pictures are small thumbnails that open the lightbox At the reading column's width a Read of a screenshot pushed the work far apart. ds/reply_picture gains Thumbs: each picture its own shape within the theme's zeron.prose.picture.thumb (240 by 160), never larger than it is, rounded and edged as reply pictures, no hover controls (its menu and the lightbox have them); several sit in a row, wrapping. ds/activity's tool rows use it, so the subagent lane does too. --- apps/zgx/ui/ds/activity.luau | 16 ++--- apps/zgx/ui/ds/reply_picture.luau | 88 ++++++++++++++++++++++- apps/zgx/ui/ds/tokens.luau | 4 +- apps/zgx/ui/tests/tool_pictures.test.luau | 45 ++++++++++-- apps/zgx/ui/tool_pictures_fixture.luau | 15 +++- 5 files changed, 147 insertions(+), 21 deletions(-) diff --git a/apps/zgx/ui/ds/activity.luau b/apps/zgx/ui/ds/activity.luau index 46300ef36..f6e9efd75 100644 --- a/apps/zgx/ui/ds/activity.luau +++ b/apps/zgx/ui/ds/activity.luau @@ -388,10 +388,10 @@ end -- (`below`: a tool with a branch follows, so the trunk runs on under this -- one; `running`: it is the one in flight, its glyph the working loader.) --- A tool's pictures under its opened row: one alone, or several on a tray, --- in a lightbox group of their own. +-- A tool's pictures under its opened row: small thumbnails in a row (the +-- theme's `zeron.prose.picture.thumb`), each opening in the lightbox among +-- the others, a lightbox group of their own. local function tool_pictures(now: () -> Tool, shown: ReplyPicture.Pictures): Node - local group = `tool:{now().id}` local function entries(): { ReplyPicture.Entry } local out = {} for _, it in now().pictures or {} do @@ -399,15 +399,7 @@ local function tool_pictures(now: () -> Tool, shown: ReplyPicture.Pictures): Nod end return out end - return ui.presence(function(): string - return if #(now().pictures or {}) > 1 then "run" else "one" - end, function(kind: string): Node - if kind == "run" then - return ReplyPicture.Run { pictures = shown, items = entries, group = group, set = entries } - end - local first = entries()[1] or { src = "", alt = "" } - return ReplyPicture.Single { pictures = shown, src = first.src, alt = first.alt, group = group, set = entries } - end, { w_full = true, min_w = 0 }) + return ReplyPicture.Thumbs { pictures = shown, items = entries, group = `tool:{now().id}`, set = entries } end local function row(now: () -> Tool, below: () -> boolean, running: () -> boolean, open_subagent: ((sub: any) -> ())?, pictures: ReplyPicture.Pictures?): Node diff --git a/apps/zgx/ui/ds/reply_picture.luau b/apps/zgx/ui/ds/reply_picture.luau index 6ebf95c20..fbd1dad81 100644 --- a/apps/zgx/ui/ds/reply_picture.luau +++ b/apps/zgx/ui/ds/reply_picture.luau @@ -115,6 +115,8 @@ local function measures(): { [string]: number } INSET = n("zeron.prose.picture.tray.inset", 4), ROW = n("zeron.prose.picture.tray.row", 300), CONTROL = n("zeron.prose.picture.control.inset", 6), + THUMB_W = n("zeron.prose.picture.thumb.w", 240), + THUMB_H = n("zeron.prose.picture.thumb.h", 160), } end @@ -373,6 +375,9 @@ type FrameProps = { fixed: boolean, -- (A run's tile: its place.) place: (() -> Place)?, + -- A thumbnail (`Thumbs`): too small for the controls over it; its + -- menu and the lightbox have them. + thumb: boolean?, } local Frame = ui.component(function(props: FrameProps): Node @@ -403,7 +408,7 @@ local Frame = ui.component(function(props: FrameProps): Node return b.w >= 120 and b.h >= 72 end local function controls_up(): boolean - return ready() and roomy() and (hovered:get() or focused:get() > 0) + return not props.thumb and ready() and roomy() and (hovered:get() or focused:get() > 0) end local inner = m.R - m.CONTROL local function open() @@ -765,6 +770,87 @@ function M.Single(props: Props): Node end)({}) end +-- A thumbnail's size for a picture of `nw` by `nh` pixels: its own shape, +-- at most the theme's thumbnail (zeron.prose.picture.thumb), never larger +-- than it is; a screenshot's shape while its size isn't known. +function M.thumb_fit(nw: number?, nh: number?): (number, number) + local m = measures() + if not nw or not nh or nw <= 0 or nh <= 0 then + return math.min(m.THUMB_W, m.THUMB_H * m.PLACE), m.THUMB_H + end + local k = math.min(1, m.THUMB_W / nw, m.THUMB_H / nh) + return math.max(1, math.floor(nw * k)), math.max(1, math.floor(nh * k)) +end + +-- Pictures as small thumbnails in a row (wrapping), each its own shape and +-- opening in the lightbox among the others: a tool's result pictures, +-- under its row, where the full size would crowd the work. +function M.Thumbs(props: RunProps): Node + return ui.component(function(): Node + local pictures = props.pictures + local entries = ui.computed(props.items) + local function pick(failed: boolean): { Entry } + local out = {} + for _, e in entries:get() do + if (now_of(pictures, e.src).state == "failed") == failed then + table.insert(out, e) + end + end + return out + end + return ui.column { + class = "md-image tool-pictures", + w_full = true, + min_w = 0, + gap = "$space.2", + ui.each(ui.computed(function(): { Entry } + return pick(false) + end), function(e: Entry): Node + local function size(): (number, number) + local p = now_of(pictures, e.src) + return M.thumb_fit(size_of(if p.state == "ready" then p.src else nil)) + end + local function place(): Place + local w, h = size() + return { x = 0, y = 0, w = w, h = h, crop = "none" } + end + return Frame { + pictures = pictures, + entry = e, + group = props.group, + set = props.set, + fixed = true, + thumb = true, + place = place, + size = size, + crop = function(): Fit + local w, h = size() + return { w = w, h = h, crop = "none" } + end, + } + end, { + key = function(e: Entry): string + return e.src + end, + direction = "row", + flex_wrap = true, + items = "start", + gap = "$space.2", + }), + ui.each(ui.computed(function(): { Entry } + return pick(true) + end), function(e: Entry): Node + return Missing { pictures = pictures, entry = e, open_link = props.open_link, link_menu = props.link_menu } + end, { + key = function(e: Entry): string + return e.src + end, + gap = "$space.2", + }), + } + end)({}) +end + -- Pictures in a row, on a tray. function M.Run(props: RunProps): Node return ui.component(function(): Node diff --git a/apps/zgx/ui/ds/tokens.luau b/apps/zgx/ui/ds/tokens.luau index df1f1b4cb..5013e665a 100644 --- a/apps/zgx/ui/ds/tokens.luau +++ b/apps/zgx/ui/ds/tokens.luau @@ -274,7 +274,8 @@ function M.setup() -- one's least tile; how long (as wide as tall, at least) and how -- wide (as tall, at most) before it's cut; a run's tray (its -- inset, so the corner is the picture's plus it, and its tallest - -- row); the controls' inset from the corner. + -- row); the controls' inset from the corner; a tool's result + -- picture's thumbnail (its widest and tallest). picture = { w = 660, h = 420, @@ -285,6 +286,7 @@ function M.setup() wide = 5, tray = { inset = 4, row = 300 }, control = { inset = 6 }, + thumb = { w = 240, h = 160 }, }, }, -- Reasoning effort, a colour a level, cool to warm as it rises: mid diff --git a/apps/zgx/ui/tests/tool_pictures.test.luau b/apps/zgx/ui/tests/tool_pictures.test.luau index ce07001ad..563f484b5 100644 --- a/apps/zgx/ui/tests/tool_pictures.test.luau +++ b/apps/zgx/ui/tests/tool_pictures.test.luau @@ -1,7 +1,9 @@ -- A tool whose result carried a picture (a Read of a screenshot: the --- engine's copy, `previewPaths`) opens to that picture, as Zeron's chip --- previews it; a click opens it in the lightbox. A Read that kept none --- opens to nothing of the kind (ui/tool_pictures_fixture.luau). +-- engine's copy, `previewPaths`) opens to it as a small thumbnail, its own +-- shape within the theme's 240 by 160 (`zeron.prose.picture.thumb`), never +-- the reading column's width; a click opens it in the lightbox. Several sit +-- in a row. A Read that kept none opens to nothing of the kind +-- (ui/tool_pictures_fixture.luau). local function work(): any local app = mount("tool_pictures_fixture.luau") @@ -9,18 +11,37 @@ local function work(): any app:frames(10) expect(app:error()).to_be_nil() -- (Settled work folds to its line: open it.) - app:click('role:button "Read 2 files"') + app:click('role:button "Read 2 files · ran 1 command"') app:frames(30) return app end -test("a read whose result kept a picture opens to it, and it opens in the lightbox", function() +-- A node's place and size in the tree, by a pattern on its line. +local function box(app: any, pattern: string): (number?, number?, number?, number?) + for line in app:tree():gmatch("[^\n]+") do + if line:find(pattern, 1, true) then + local x, y, w, h = line:match("(%-?[%d%.]+),(%-?[%d%.]+) ([%d%.]+)x([%d%.]+)") + if x then + return tonumber(x), tonumber(y), tonumber(w), tonumber(h) + end + end + end + return nil +end + +test("a read whose result kept a picture opens to a thumbnail of it, and it opens in the lightbox", function() local app = work() expect(app:has('"Open test.png"')).to_be(false) app:click('role:button "Read test.png"') app:frames(30) expect(app:error()).to_be_nil() expect(app:has('"Open test.png"')).to_be(true) + -- (868 by 617: as tall as the thumbnail allows, its shape kept.) + local _, _, w, h = box(app, '"Open test.png"') + expect(w).never.to_be_nil() + expect(h <= 160.5).to_be(true) + expect(w <= 240.5).to_be(true) + expect(math.abs(w / h - 868 / 617) < 0.02).to_be(true) app:click('"Open test.png"') app:frames(60) local open = app:state("ds/lightbox.luau::open") @@ -28,6 +49,20 @@ test("a read whose result kept a picture opens to it, and it opens in the lightb expect(open.items[open.index].src).to_be("assets/composer/shell.png") end) +test("several pictures are thumbnails side by side", function() + local app = work() + app:click('role:button "Ran shoot both"') + app:frames(30) + expect(app:error()).to_be_nil() + local lx, ly, lw, lh = box(app, '"Open left.png"') + local rx, ry = box(app, '"Open right.png"') + expect(lx).never.to_be_nil() + expect(rx).never.to_be_nil() + expect(ly).to_be(ry) + expect(rx > lx + lw - 0.5).to_be(true) + expect(lh <= 160.5).to_be(true) +end) + test("a read that kept no picture shows none", function() local app = work() app:click('role:button "Read old.png"') diff --git a/apps/zgx/ui/tool_pictures_fixture.luau b/apps/zgx/ui/tool_pictures_fixture.luau index 3dbfaff5f..dafe00d1f 100644 --- a/apps/zgx/ui/tool_pictures_fixture.luau +++ b/apps/zgx/ui/tool_pictures_fixture.luau @@ -1,6 +1,7 @@ -- A reply's work (ds/activity) with a Read of a picture whose result the --- engine kept a copy of (its `pictures`, the tool's `previewPaths`), and a --- Read of another that kept none, for tests/tool_pictures.test.luau. The +-- engine kept a copy of (its `pictures`, the tool's `previewPaths`), a Read +-- of another that kept none, and a command whose result kept two, for +-- tests/tool_pictures.test.luau. The -- pictures are read from where the transcript reads an agent's shown ones -- (stubbed: each path is the picture, nothing read). -- Here, beside app.luau, as a mounted script loads modules from its own @@ -28,6 +29,16 @@ local TOOLS = { pictures = { { path = "assets/composer/shell.png", name = "test.png" } }, }, { id = "r2", kind = "read", verb = "Read", target = "old.png" }, + { + id = "r3", + kind = "exec", + verb = "Ran", + target = "shoot both", + pictures = { + { path = "assets/composer/shell.png", name = "left.png" }, + { path = "assets/composer/transcript.png", name = "right.png" }, + }, + }, } return ui.column { From bce5f058866a8c1be1fe8d8b1694e6173c9726f5 Mon Sep 17 00:00:00 2001 From: Daniel Katulevskiy Date: Tue, 3 Nov 2026 18:03:07 -0800 Subject: [PATCH 05/81] Tool images: a compact thumbnail that opens the lightbox A Read of a screenshot filled the open chip with a 220 pt tall picture up to 560 wide. Each picture is now a thumbnail of its own shape within THUMB_FRAME (240 by 160), never scaled up, rounded and edged as reply pictures, in a row; the decode bound follows at 2x (480 by 320). A click opens the full file in the lightbox, read from the chat's host as a sent picture is (once it has loaded, if it was still coming). --- crates/ui/src/tool_images.rs | 42 +++++++++++++++-- crates/ui/src/transcript.rs | 88 +++++++++++++++++++++++++++++++----- 2 files changed, 115 insertions(+), 15 deletions(-) diff --git a/crates/ui/src/tool_images.rs b/crates/ui/src/tool_images.rs index 1281677f0..ce8a8acac 100644 --- a/crates/ui/src/tool_images.rs +++ b/crates/ui/src/tool_images.rs @@ -20,9 +20,27 @@ use crate::state::EngineHandle; /// Previews per chip; further paths stay in the text output. pub(crate) const MAX_IMAGES_PER_TOOL: usize = 4; -/// Decoded thumbnail bounds: the 560×220 frame at 2× density (≤ 1.9 MiB -/// BGRA per copy). -const THUMB_MAX: (u32, u32) = (1120, 440); +/// A tool result's picture in its open chip: a thumbnail at most this wide +/// and tall (points), its own shape, never larger than its pixels. A click +/// opens it in the lightbox, so the chip stays compact. +pub(crate) const THUMB_FRAME: (f32, f32) = (240.0, 160.0); +/// Decoded thumbnail bounds: [`THUMB_FRAME`] at 2× density (≤ 600 KiB BGRA +/// per copy). +const THUMB_MAX: (u32, u32) = (480, 320); + +/// A thumbnail's size in points for a picture of `pixels` (as decoded, at +/// most [`THUMB_MAX`], which keeps its shape): within [`THUMB_FRAME`], +/// never scaled up. A screenshot's shape (16:10) while it isn't known. +pub(crate) fn thumb_size(pixels: Option<(f32, f32)>) -> (f32, f32) { + let (frame_w, frame_h) = THUMB_FRAME; + match pixels { + Some((w, h)) if w > 0.0 && h > 0.0 => { + let k = (frame_w / w).min(frame_h / h).min(1.0); + ((w * k).round().max(1.0), (h * k).round().max(1.0)) + } + _ => (frame_w, (frame_w / 1.6).min(frame_h)), + } +} /// Collapsed chips keep pixels this long, so open/close spam never re-decodes. pub(crate) const RELEASE_GRACE: Duration = Duration::from_millis(600); /// Expanded-but-offscreen previews may be dropped past this (CPU + GPU bytes). @@ -675,6 +693,24 @@ mod tests { ); } + #[test] + fn thumbnails_fit_the_frame_keep_their_shape_and_never_grow() { + let (fw, fh) = THUMB_FRAME; + // A screenshot, larger than the frame: as tall as it allows. + let (w, h) = thumb_size(Some((1884.0, 2066.0))); + assert!(w <= fw && h <= fh, "{w}x{h}"); + assert_eq!(h, fh); + assert!((w / h - 1884.0 / 2066.0).abs() < 0.01); + // A wide strip: as wide as it allows. + let (w, h) = thumb_size(Some((480.0, 60.0))); + assert_eq!((w, h), (fw, 30.0)); + // A small icon stays its own size. + assert_eq!(thumb_size(Some((32.0, 24.0))), (32.0, 24.0)); + // Unknown yet: a placeholder inside the frame. + let (w, h) = thumb_size(None); + assert!(w <= fw && h <= fh); + } + #[test] fn stored_preview_paths_replace_the_named_file() { let call = ToolCall::ReadFile { diff --git a/crates/ui/src/transcript.rs b/crates/ui/src/transcript.rs index 1e9fe4481..5d61d36fa 100644 --- a/crates/ui/src/transcript.rs +++ b/crates/ui/src/transcript.rs @@ -2347,10 +2347,12 @@ pub fn detail_height(detail: &ToolDetail) -> f32 { DETAIL_SEPARATOR + body } -/// Tool image preview frame height; widths follow each image's aspect. -const TOOL_IMAGE_HEIGHT: f32 = 220.0; -const TOOL_IMAGE_MAX_WIDTH: f32 = 560.0; +/// Tool image thumbnails sit in a strip as tall as the tallest one can be +/// ([`crate::tool_images::THUMB_FRAME`]); each is its own shape inside it. +const TOOL_IMAGE_HEIGHT: f32 = crate::tool_images::THUMB_FRAME.1; const TOOL_IMAGE_PAD: f32 = 8.0; +/// Reply pictures' corner (`reply_images::frame`), for the thumbnails too. +const TOOL_IMAGE_RADIUS: f32 = 12.0; /// Analytic height an open chip's image strip adds (separator + frames). const TOOL_IMAGE_STRIP_HEIGHT: f32 = DETAIL_SEPARATOR + TOOL_IMAGE_HEIGHT + 2.0 * TOOL_IMAGE_PAD; @@ -3532,6 +3534,9 @@ pub struct Transcript { /// Focused while the lightbox is open so Escape reaches it. attachment_preview_focus: gpui::FocusHandle, attachment_preview_return_focus: Option, + /// A tool thumbnail clicked while its full picture still loads: it + /// opens in the lightbox once it's here (`(devices, path, name)`). + pending_tool_image: Option<(Vec, String, SharedString)>, /// Mermaid fences drawn as diagrams in settled Markdown blocks. Each /// row's media handler requests its fences while laying out, so only /// painted diagrams are rendered (see [`MermaidCache`]). @@ -3864,6 +3869,7 @@ impl Transcript { attachment_preview: None, attachment_preview_focus: cx.focus_handle(), attachment_preview_return_focus: None, + pending_tool_image: None, diagrams: Rc::default(), diagram_media: None, diagram_worker: None, @@ -6376,29 +6382,55 @@ impl Transcript { ) -> AnyElement { use crate::tool_images::Thumb; let devices: Rc<[String]> = self.image_host(cx).0.into_iter().collect(); + let accent = theme.accent; let frames = images.into_iter().enumerate().map(|(ix, (path, thumb))| { let name = SharedString::from(file_badge_name(&path).to_owned()); - let width = match &thumb { - Thumb::Ready { aspect, .. } => { - (TOOL_IMAGE_HEIGHT * aspect).clamp(48.0, TOOL_IMAGE_MAX_WIDTH) + let (width, height) = crate::tool_images::thumb_size(match &thumb { + Thumb::Ready { image, aspect } => { + let size = image.size(0); + Some(if size.width.0 > 0 && size.height.0 > 0 { + (size.width.0 as f32, size.height.0 as f32) + } else { + (*aspect * TOOL_IMAGE_HEIGHT, TOOL_IMAGE_HEIGHT) + }) } - _ => TOOL_IMAGE_HEIGHT * 4.0 / 3.0, - }; + _ => None, + }); let menu_devices = devices.clone(); + let open_devices = devices.to_vec(); + let open_path = path.clone(); + let open_name = name.clone(); + let ready = matches!(thumb, Thumb::Ready { .. }); let frame = div() .id(SharedString::from(format!("{chip}-img{ix}"))) .w(px(width)) - .h(px(TOOL_IMAGE_HEIGHT)) + .h(px(height)) .flex_none() .flex() .items_center() .justify_center() - .rounded(px(8.0)) + .rounded(px(TOOL_IMAGE_RADIUS)) .overflow_hidden() .border_1() .border_color(crate::theme::hairline(0.07)) .bg(crate::theme::ink(0.035)) .aria_label(name.clone()) + .when(ready, |frame| { + frame + .role(gpui::Role::Button) + .tab_index(0) + .cursor_pointer() + .focus_visible(move |style| style.border_2().border_color(accent)) + .on_click(cx.listener(move |this, _, window, cx| { + this.open_tool_image( + open_devices.clone(), + open_path.clone(), + open_name.clone(), + window, + cx, + ); + })) + }) .on_mouse_down( gpui::MouseButton::Right, cx.listener(move |this, event: &gpui::MouseDownEvent, window, cx| { @@ -6418,8 +6450,8 @@ impl Transcript { Thumb::Ready { image, .. } => frame.child( img(image) .w(px(width - 2.0)) - .h(px(TOOL_IMAGE_HEIGHT - 2.0)) - .rounded(px(7.0)) + .h(px(height - 2.0)) + .rounded(px(TOOL_IMAGE_RADIUS - 1.0)) .object_fit(ObjectFit::Contain), ), Thumb::Loading => frame @@ -6455,6 +6487,7 @@ impl Transcript { .when(!collapses, |strip| strip.px(px(TOOL_IMAGE_PAD))) .flex() .flex_row() + .items_start() .gap(px(TOOL_IMAGE_PAD)) .overflow_x_scroll() .children(frames), @@ -7338,6 +7371,31 @@ impl Transcript { } } + /// Open a tool thumbnail's picture in the lightbox: its full file, read + /// from the chat's host as a sent picture is. Still loading, it opens + /// once it's here (`pending_tool_image`); unreadable, nothing opens. + fn open_tool_image( + &mut self, + devices: Vec, + path: String, + name: SharedString, + window: &mut Window, + cx: &mut Context, + ) { + use crate::attachments::AttachmentSnapshot; + match self.attachment_state(&devices, &path, None, cx) { + AttachmentSnapshot::Loaded(loaded) => { + self.pending_tool_image = None; + let preview = crate::attachments::PreviewImage::new(name, loaded.image.clone()); + self.open_image_preview(preview, window, cx); + } + AttachmentSnapshot::Loading => { + self.pending_tool_image = Some((devices, path, name)); + } + _ => self.pending_tool_image = None, + } + } + /// Open an image in the shared lightbox; Escape returns focus here. fn open_image_preview( &mut self, @@ -11509,6 +11567,12 @@ impl Render for Transcript { .flatten() .map(|loading| self.render_loader(loading, cx)); let root = root.children(loader); + // A tool thumbnail clicked while its picture loaded: open it now. + if self.attachment_preview.is_none() + && let Some((devices, path, name)) = self.pending_tool_image.take() + { + self.open_tool_image(devices, path, name, window, cx); + } // Full-size viewer for a clicked user-bubble thumbnail // (AttachmentPreviewDialog: bare lightbox, click closes). if let Some(preview) = self.attachment_preview.clone() { From cd7694028f536c6f1124fcfde71faaf0e8ac6a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gurruchaga?= Date: Sun, 8 Nov 2026 18:18:09 -0300 Subject: [PATCH 06/81] test(zgx): characterize composer focus and layout transitions --- apps/zgx/docs/composer-characterization.md | 145 ++++++ apps/zgx/src/route_motion_tests.rs | 474 +++++++++++++++--- apps/zgx/ui/composer_motion_fixture.luau | 11 + apps/zgx/ui/ds/composer.luau | 5 +- apps/zgx/ui/ds/composer_clock.luau | 4 + apps/zgx/ui/screens/chat/composer.luau | 13 +- apps/zgx/ui/templates/zeron-v2/.base.json | 2 +- .../zeron-v2/screens/chat/composer.luau | 9 +- 8 files changed, 579 insertions(+), 84 deletions(-) create mode 100644 apps/zgx/docs/composer-characterization.md create mode 100644 apps/zgx/ui/composer_motion_fixture.luau create mode 100644 apps/zgx/ui/ds/composer_clock.luau diff --git a/apps/zgx/docs/composer-characterization.md b/apps/zgx/docs/composer-characterization.md new file mode 100644 index 000000000..4e4f26d35 --- /dev/null +++ b/apps/zgx/docs/composer-characterization.md @@ -0,0 +1,145 @@ +# Composer focus and layout baseline + +This is characterization for the composer refactor, **not a confirmed +reproduction or fix of the intermittent glitch when clicking or typing**. +Production deadline reads delegate to `ds/composer_clock.now`, whose default +is `ui.now`. No layout rules or framework behavior change in this baseline. + +## Automated observations + +Run from the repository root: + +```sh +cargo test --locked --manifest-path apps/zgx/Cargo.toml --bin zeron-zgx route_motion_tests +``` + +`src/route_motion_tests.rs` mounts the real shell via +`ui/composer_motion_fixture.luau`. Each observed frame records the editor's +node identity, focused node, selection in UTF-8 bytes, committed draft, +transformed field/composer rectangles, ancestor opacity and scene visibility. +Assertion failures include the frame trace for the current interaction. +To retain every observed sample on success too: + +```sh +ZERON_ZGX_COMPOSER_TRACE=1 cargo test --locked --manifest-path apps/zgx/Cargo.toml --bin zeron-zgx route_motion_tests -- --nocapture > /tmp/zgx-composer-trace.log 2>&1 +``` + +Trace lines begin with `composer-trace` and include the test name so parallel +tests can be distinguished. + +The destination rectangles come from a second shell receiving the same +inputs with Reduce motion enabled. They are an independently laid-out +reference endpoint, not direct access to the animated composer's private +target signals. This avoids reimplementing the composer's geometry formulas +in tests. It also cannot detect a calculation error shared by both shells. + +The tests cover: + +- New-chat and session composers: focus loss, a real click to regain focus, + and a selection retained without editing or moving/resizing the field. +- Character-by-character typing through the compact/expanded boundary, + then backspacing until compact again. The boundary is discovered with + the current font/layout rather than an assumed character count. +- Multiline growth and shrink in both modes, including more typing while + growth is still in progress. The test checks that it actually interrupts + an intermediate geometry, rather than typing after the animation ends. +- New-chat/session route changes: immediate destination geometry, a + retained editor node, and separate session drafts restored on return. + +`TestApp` supplies 60 Hz frames and a controlled timer clock. `ui.now()` +normally uses wall time, so the dedicated fixture replaces `composer_clock.now` with a state +advanced by the tests to the same logical elapsed time. The override is +confined to that fixture's VM and uses `peek()` to avoid making time a new +reactive layout dependency. These tests use no sleeps. The ordinary shell +fixture and the app retain their real clock. + +This is headless scene validation. Scene visibility and opacity do **not** +prove that the GPU painted the text or that a presented frame was correct. + +## Visual reproduction procedure + +Status at implementation: the original visual glitch has not been +reproduced by these headless tests. A valid standalone visual comparison +and the macOS comparison have not been executed in this change. The following is a repeatable diagnostic +procedure, not a claim that these steps reproduce it on every machine. +macOS native-caret/Metal comparison requires a separate run on macOS. + +Use the standalone **ZGX runner built from the revision pinned in +`apps/zgx/Cargo.toml`**, with the ordinary shell fixture, not the controlled +clock fixture. Pointer input, timers and presentation then run normally: + +```sh +zgx-run --inspect apps/zgx/ui/shell_fixture.luau +``` + +The standalone runner deliberately omits Zeron's engine host. Launching the +same shell through `zeron-zgx fixture=shell_fixture.luau` is not equivalent: +its host can overwrite the seeded sessions with engine data, or mark the +account signed out after a connection timeout. Do not use that launch as +an isolated visual baseline. Record the actual fonts/fallback used by the +runner, since the Zeron host also registers bundled fonts. + +Use a 1200 × 800 logical-point content area at 100% zoom. The fixture window +restores its size, so resize it explicitly and record the actual size. +Start with Zeron dark, transparency on and Reduce motion off. Record display +scale and refresh rate as well. The fixture has made-up sessions and does +not require sending a real agent turn. Its menu command `Fixture: dark` +selects the dark theme (`fixture.dark` through the command/inspect API). + +Perform this sequence while recording video at the display's available +frame rate; before/after screenshots alone miss a one-frame glitch: + +1. Open **Fix the build**, click its empty composer, click the conversation, + then click the composer again. Repeat five times without entering text. +2. Type `hello composer`, click at different positions, select `composer`, + and move the caret with the arrow keys. Do not submit the message. +3. Select all and clear. Type `W` one character at a time until it expands; + record the character count at that boundary. Delete characters one at a + time until it folds. Repeat close to the measured boundary. +4. Select all and enter six lines: `one`, `two`, `three`, `four`, `five`, + `six`. While it is visibly growing, add a seventh line, `seven`. + Replace the draft with `short` while observing the shrinking field. +5. Open **New chat** and repeat steps 1–4 in the expanded composer. +6. Return to **Fix the build**, then **Write the docs**, give them different + drafts and switch between them. Check that drafts, focus and geometry + correspond to the session shown. + +Repeat the relevant sequence at 90% zoom, then with transparency off and +with Reduce motion on, changing one condition per run. For a report from +an installed beta, also record its exact version, active template/plugins +and reproduce in that app: a fixture is useful isolation, not identical to +the full live beta environment. + +## Diagnostic comparisons + +First capture the normal run without either environment variable. On +macOS, compare the same sequence with the native caret disabled: + +```sh +ZGX_NATIVE_CARET=0 zgx-run --inspect apps/zgx/ui/shell_fixture.luau +``` + +Run the partial-vs-full repaint comparison separately, keeping stderr: + +```sh +ZGX_DAMAGE_CHECK=1 zgx-run --inspect apps/zgx/ui/shell_fixture.luau 2> /tmp/zgx-composer-damage.log +``` + +The damage check compares partial output to a full repaint and reports +stale pixels. It forces additional rendering/readback, changes timing and +may mask the glitch. An empty log does not rule out presentation errors; +a changed symptom in this mode is not proof of its cause. Do not enable +either switch permanently or combine them in the first comparison. + +## Evidence to attach to each run + +Record revision/build and platform, content size, zoom, theme, transparency, +Reduce motion, display scale/refresh rate, active template/plugins, switches +and input sequence (including wrap count). Attach the video with timestamps +and the relevant stderr excerpt. Label each run **observed**, **not +observed**, or **not executed**. If observed, describe what changed without +assuming it disappeared: text, cursor, geometry, background or whole window. + +Keep geometry/focus/selection failures separate from pixel or presentation +failures. A passing headless baseline alone does not close the original +visual report. diff --git a/apps/zgx/src/route_motion_tests.rs b/apps/zgx/src/route_motion_tests.rs index bcbccf94b..8725883fa 100644 --- a/apps/zgx/src/route_motion_tests.rs +++ b/apps/zgx/src/route_motion_tests.rs @@ -1,84 +1,416 @@ -//! The composer between a new chat and a session (ui/screens/chat/composer, -//! mounted headless on ui/shell_fixture.luau): another chat opened or a new -//! chat puts it where it goes at once, no fade and no slide. (The Lua tests can't read an opacity: -//! ui/tests/shell_route_motion.test.luau has the panel's side of it.) +//! Composer focus and layout transitions on the real shell fixture. +//! The fixture controls composer deadlines alongside TestApp's frame/timer clocks. -use std::time::Duration; +use zgx::{ + input::{InputEvent, Key}, + scene::{NodeId, Rect}, +}; +use zgx_script::{ScriptHost, Value, testing::TestApp}; -use zgx_script::{ScriptHost, testing::TestApp}; +// Characterization, not a reproduction of the reported visual glitch. +// These snapshots observe the scene, not GPU pixels. See +// docs/composer-characterization.md for the separate visual procedure. +const FIELD: &str = "#chat-composer >> role:textbox"; +const BUILD: &str = "#shell-sidebar >> role:button \"Fix the build\""; +const DOCS: &str = "#shell-sidebar >> role:button \"Write the docs\""; +const NEW: &str = "#shell-sidebar >> role:button \"New chat\""; -fn mount() -> TestApp { - // (This checkout's `ui/`, or `ZERON_ZGX_TEST_UI`'s: another copy, to see - // the test fail on code from before a fix.) - let ui = std::env::var_os("ZERON_ZGX_TEST_UI") - .map(std::path::PathBuf::from) - .unwrap_or_else(|| std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("ui")); - let path = ui.join("shell_fixture.luau"); - let app = TestApp::new(|runtime| ScriptHost::new(runtime, path)); - app.ui.scene.borrow_mut().resize(1200., 800.); - app.frames(30); - app +struct Shell { + app: TestApp, + frame: u32, } -/// A frame and a little of the clock, `n` times: each look at the dock's -/// opacity, the composer's top and its height (the pill's, with what hangs -/// off it). -fn watch(app: &mut TestApp, n: usize) -> Vec<(f32, f32, f32)> { - (0..n) - .map(|_| { - app.frames(1); - std::thread::sleep(Duration::from_millis(12)); - let b = app.bounds("chat-composer"); - (app.opacity("chat-composer-dock"), b.y, b.height) - }) - .collect() +impl Shell { + fn new(reduced: bool) -> Self { + let ui = std::env::var_os("ZERON_ZGX_TEST_UI") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("ui")); + let app = TestApp::new(|runtime| { + ScriptHost::new(runtime, ui.join("composer_motion_fixture.luau")) + }); + app.ui.scene.borrow_mut().resize(1200., 800.); + let motion = app + .host + .states() + .into_iter() + .find(|(name, _)| name.ends_with("::motion_reduce")) + .expect("the shell's motion preference") + .0; + app.host.set_state(&motion, Value::Bool(reduced)).unwrap(); + let mut shell = Self { app, frame: 0 }; + shell.frames(60); + shell + } + + fn frames(&mut self, n: usize) { + for _ in 0..n { + self.frame += 1; + self.app + .host + .set_state( + "fixture_composer_now", + Value::Num(f64::from(self.frame) * 1000. / 60.), + ) + .unwrap(); + self.app.frame(); + assert_eq!( + self.app.error(), + None, + "script error at frame {}", + self.frame + ); + } + } + + fn geometry(&mut self) -> Geometry { + let composer = self.app.find("#chat-composer").expect("composer"); + let field = self.app.find(FIELD).expect("draft field"); + let scene = self.app.ui.scene.borrow(); + Geometry { + composer: scene.transformed_bounds(composer), + field: scene.transformed_bounds(field), + } + } } -/// Every look after a switch: shown in full (no fade), at its final height -/// (nothing growing or shrinking), and already where it ends (`to`), give -/// or take a frame of its size being measured (a quarter of the way -/// between the two places; a slide would pass through the rest). -fn at_once(seen: &[(f32, f32, f32)], from: f32, to: f32, what: &str) { - let way = (from - to).abs() / 4.; - let tall = seen.last().unwrap().2; - for (i, (opacity, y, h)) in seen.iter().enumerate() { - assert!((opacity - 1.0).abs() < 0.01, "{what}: faded, frame {}: opacity {opacity}", i + 1); - assert!((h - tall).abs() <= 0.5, "{what}: frame {} {h} tall, growing to {tall}", i + 1); - assert!((y - to).abs() <= way, "{what}: frame {} at {y}, not yet where it goes ({to}, from {from})", i + 1); +#[derive(Clone, Copy, Debug)] +struct Geometry { + composer: Rect, + field: Rect, +} + +#[derive(Debug)] +struct Sample { + frame: u32, + editor: NodeId, + focused: Option, + selection: (usize, usize), + text: String, + presented: Geometry, + target: Geometry, + opacity: f32, + visible: bool, +} + +struct Composer { + shown: Shell, + target: Shell, + editor: NodeId, + trace: Vec, + print_trace: bool, +} + +impl Composer { + fn new(session: bool) -> Self { + let mut shown = Shell::new(false); + let mut target = Shell::new(true); + if session { + shown.app.click(BUILD); + target.app.click(BUILD); + shown.frames(60); + target.frames(60); + } + let field = shown.app.find(FIELD).unwrap(); + let editor = shown.app.ui.editor_within(field).unwrap().node; + Self { + shown, + target, + editor, + trace: Vec::new(), + print_trace: std::env::var_os("ZERON_ZGX_COMPOSER_TRACE").is_some_and(|v| v != "0"), + } + } + + /// Same input in both shells. The reduced-motion shell supplies the + /// destination through real layout rather than duplicating its formulas. + fn act(&mut self, action: impl Fn(&mut TestApp)) { + action(&mut self.shown.app); + action(&mut self.target.app); + self.target.frames(2); + } + + fn watch(&mut self, n: usize) { + for _ in 0..n { + self.shown.frames(1); + let presented = self.shown.geometry(); + let target = self.target.geometry(); + let field = self.shown.app.find(FIELD).unwrap(); + let editor = self.shown.app.ui.editor_within(field).unwrap(); + let scene = self.shown.app.ui.scene.borrow(); + let opacity = std::iter::once(editor.node) + .chain(scene.ancestors(editor.node)) + .map(|id| scene.effects(id).opacity) + .product(); + let sample = Sample { + frame: self.shown.frame, + editor: editor.node, + focused: self.shown.app.ui.input.focused(), + selection: editor.selection(), + text: editor.editor.borrow().text().into_owned(), + presented, + target, + opacity, + visible: scene.visible_bounds(editor.node).is_some(), + }; + drop(scene); + self.trace.push(sample); + let s = self.trace.last().unwrap(); + if self.print_trace { + eprintln!( + "composer-trace {} {s:?}", + std::thread::current().name().unwrap_or("unnamed") + ); + } + assert_eq!(s.editor, self.editor, "editor replaced: {:?}", self.trace); + assert!( + s.visible && s.opacity > 0.99, + "editor hidden: {:?}", + self.trace + ); + for r in [ + s.presented.composer, + s.presented.field, + s.target.composer, + s.target.field, + ] { + assert!( + r.x.is_finite() + && r.y.is_finite() + && r.width.is_finite() + && r.height.is_finite() + && r.width > 0. + && r.height > 0., + "invalid geometry: {:?}", + self.trace + ); + } + } + } + + fn last(&self) -> &Sample { + self.trace.last().expect("observed frames") + } + + fn draft(&self, text: &str, selection: (usize, usize)) { + for s in &self.trace { + assert_eq!( + s.text, text, + "draft changed at frame {}: {:?}", + s.frame, self.trace + ); + assert_eq!( + s.selection, selection, + "selection changed: {:?}", + self.trace + ); + assert_eq!(s.focused, Some(self.editor), "focus lost: {:?}", self.trace); + } + } + + fn at_target(&self) { + let s = self.last(); + for (a, b) in [ + (s.presented.composer, s.target.composer), + (s.presented.field, s.target.field), + ] { + assert!( + (a.x - b.x).abs() < 1. + && (a.y - b.y).abs() < 1. + && (a.width - b.width).abs() < 1. + && (a.height - b.height).abs() < 1., + "did not reach reduced-motion target: {:?}", + self.trace + ); + } + } +} + +fn select(app: &mut TestApp, anchor: usize, focus: usize) { + let field = app.find(FIELD).unwrap(); + app.ui + .editor_within(field) + .unwrap() + .set_selection(anchor, focus); +} + +fn type_at_caret(app: &mut TestApp, text: &str) { + // TestApp::type_text clicks again; dispatch directly so typing really + // preserves the current caret/selection, including during a transition. + app.ui.dispatch(InputEvent::Text(text.into())); + app.settle(); +} + +#[test] +fn composer_click_and_focus_preserve_geometry_and_the_selected_draft() { + for session in [false, true] { + let mut c = Composer::new(session); + c.watch(1); + let empty = c.last().presented; + c.act(|app| app.click(FIELD)); + c.trace.clear(); + c.watch(6); + c.draft("", (0, 0)); + for s in &c.trace { + assert_eq!( + s.presented.composer, empty.composer, + "empty click resized composer: {s:?}" + ); + assert_eq!( + s.presented.field, empty.field, + "empty click moved field: {s:?}" + ); + } + c.trace.clear(); + c.act(|app| app.fill(FIELD, "hello composer")); + c.watch(40); + c.at_target(); + let before = c.last().presented; + c.trace.clear(); + c.act(|app| { + app.ui.input.focus(&app.ui.scene, None); + }); + c.watch(2); + assert_eq!(c.last().focused, None); + c.act(|app| app.click(FIELD)); + c.trace.clear(); + c.watch(2); + let clicked = c.last().selection; + c.draft("hello composer", clicked); + for s in &c.trace { + assert_eq!(s.presented.field, before.field, "click moved field: {s:?}"); + } + c.act(|app| select(app, 1, 8)); + c.trace.clear(); + c.watch(40); + c.draft("hello composer", (1, 8)); + for s in &c.trace { + assert_eq!( + s.presented.composer, before.composer, + "focus resized composer: {s:?}" + ); + assert_eq!(s.presented.field, before.field, "focus moved field: {s:?}"); + } + c.at_target(); + } +} + +#[test] +fn composer_character_typing_crosses_the_wrap_boundary_and_folds_back() { + let mut c = Composer::new(true); + c.act(|app| app.click(FIELD)); + c.watch(1); + let compact = c.last().target.composer.height; + let mut text = String::new(); + // Find the boundary with the actual font, without copying width rules. + while c.last().target.composer.height <= compact + 1. { + assert!(text.len() < 256, "never expanded: {:?}", c.trace); + text.push('W'); + c.act(|app| type_at_caret(app, "W")); + c.trace.clear(); + c.watch(1); + c.draft(&text, (text.len(), text.len())); + } + c.watch(30); + c.at_target(); + assert!(c.last().presented.composer.height > compact + 20.); + while !text.is_empty() { + c.act(|app| app.key(Key::Backspace)); + text.pop(); + c.trace.clear(); + c.watch(1); + c.draft(&text, (text.len(), text.len())); + } + c.watch(40); + c.at_target(); + assert!((c.last().presented.composer.height - compact).abs() < 1.); +} + +#[test] +fn composer_growth_shrink_and_typing_during_a_transition_keep_the_editor() { + for session in [false, true] { + let mut c = Composer::new(session); + c.act(|app| app.click(FIELD)); + c.watch(1); + let short = c.last().presented.composer.height; + let first = "one\ntwo\nthree\nfour\nfive\nsix"; + c.act(|app| type_at_caret(app, first)); + c.trace.clear(); + c.watch(3); + c.draft(first, (first.len(), first.len())); + let mid = c.last(); + assert!( + mid.presented.composer.height > short + 1. + && mid.presented.composer.height < mid.target.composer.height - 1., + "must interrupt an actual growth transition: {:?}", + c.trace + ); + c.act(|app| type_at_caret(app, "\nseven")); + c.trace.clear(); + c.watch(40); + let text = format!("{first}\nseven"); + c.draft(&text, (text.len(), text.len())); + c.at_target(); + let tall = c.last().presented.composer.height; + c.act(|app| { + select(app, 0, text.len()); + type_at_caret(app, "short"); + }); + c.trace.clear(); + c.watch(3); + assert!( + c.last().presented.composer.height < tall - 1. + && c.last().presented.composer.height > c.last().target.composer.height + 1., + "shrink should ease: {:?}", + c.trace + ); + c.watch(40); + c.draft("short", (5, 5)); + c.at_target(); } } #[test] fn a_new_chat_and_a_session_switch_the_composer_at_once() { - let mut app = mount(); - app.click("#shell-sidebar >> role:button \"Fix the build\""); - watch(&mut app, 60); - let docked = app.bounds("chat-composer").y; - assert!((app.opacity("chat-composer-dock") - 1.0).abs() < 0.01, "the composer shows, docked"); - - // To a new chat: mid-page from the first frame, never faded. - app.click("#shell-sidebar >> role:button \"New chat\""); - let seen = watch(&mut app, 60); - let hero = app.bounds("chat-composer").y; - assert!(hero < docked - 50., "it stands mid-page on a new chat ({hero} vs {docked})"); - at_once(&seen, docked, hero, "to a new chat"); - - // And back to the session: docked from the first frame. - app.click("#shell-sidebar >> role:button \"Fix the build\""); - let seen = watch(&mut app, 60); - assert!((app.bounds("chat-composer").y - docked).abs() <= 1., "docked again"); - at_once(&seen, hero, docked, "back to the session"); - - // Growing in the same chat (lines typed) still eases: seen on its way. - std::thread::sleep(Duration::from_millis(500)); - let short = app.bounds("chat-composer").height; - app.fill("#chat-composer >> role:textbox", "one\ntwo\nthree\nfour"); - let seen = watch(&mut app, 40); - let tall = seen.last().unwrap().2; - assert!(tall > short + 20., "the lines opened it out ({short} → {tall})"); - assert!( - seen.iter().any(|(_, _, h)| *h > short + 1. && *h < tall - 1.), - "it grew at once, not eased ({short} → {tall}): {:?}", - seen.iter().map(|s| s.2).collect::>() - ); + let mut c = Composer::new(true); + c.watch(1); + let docked = c.last().presented.composer.y; + for selector in [NEW, BUILD] { + c.act(|app| app.click(selector)); + c.trace.clear(); + c.watch(30); + for s in &c.trace { + assert!( + (s.presented.composer.y - s.target.composer.y).abs() < 1. + && (s.presented.composer.height - s.target.composer.height).abs() < 1., + "route switch did not snap: {:?}", + c.trace + ); + } + c.at_target(); + if selector == NEW { + assert!(c.last().presented.composer.y < docked - 50.); + } else { + assert!((c.last().presented.composer.y - docked).abs() < 1.); + } + } + c.act(|app| app.fill(FIELD, "build draft")); + c.watch(40); + c.act(|app| app.click(DOCS)); + c.trace.clear(); + c.watch(30); + assert_eq!(c.last().text, ""); + c.act(|app| app.fill(FIELD, "docs draft")); + c.watch(40); + c.act(|app| app.click(BUILD)); + c.trace.clear(); + c.watch(30); + assert_eq!(c.last().text, "build draft"); + c.act(|app| { + app.click(FIELD); + select(app, 0, 5); + }); + c.trace.clear(); + c.watch(30); + c.draft("build draft", (0, 5)); + c.at_target(); } diff --git a/apps/zgx/ui/composer_motion_fixture.luau b/apps/zgx/ui/composer_motion_fixture.luau new file mode 100644 index 000000000..48a6e13eb --- /dev/null +++ b/apps/zgx/ui/composer_motion_fixture.luau @@ -0,0 +1,11 @@ +-- Headless route-motion tests drive this clock alongside TestApp's frames. +-- ui.now normally reads wall time, unlike the harness's frame/timer clocks. +-- Keep the override in this fixture's VM; the app and visual fixtures use +-- the real clock. Peek so advancing time alone starts no reactive work. +local now = ui.state("fixture_composer_now", 0) +local clock = require("@ds/composer_clock") +clock.now = function(): number + return now:peek() +end + +return require("./shell_fixture") diff --git a/apps/zgx/ui/ds/composer.luau b/apps/zgx/ui/ds/composer.luau index 60302ba00..f505a7df2 100644 --- a/apps/zgx/ui/ds/composer.luau +++ b/apps/zgx/ui/ds/composer.luau @@ -54,6 +54,7 @@ local mentions = require("@self/composer_mentions") local locale = require("@self/locale") local context_usage = require("@self/context_usage") local motion = require("@self/motion") +local clock = require("@ds/composer_clock") local EditorMenu = require("@self/editor_menu") -- What the model's context holds, as its agent reports it: the tokens in @@ -1191,10 +1192,10 @@ return ui.component(function(props: Props): Node local n = props.snap:get() if n ~= snap_seen then snap_seen = n - snap_until = ui.now() + SNAP_MS + snap_until = clock.now() + SNAP_MS end end - return motion.reduced_now() or ui.now() < snap_until + return motion.reduced_now() or clock.now() < snap_until end local function eased(src: ReadOnlyState): State local value = ui.use_state(src:peek()) diff --git a/apps/zgx/ui/ds/composer_clock.luau b/apps/zgx/ui/ds/composer_clock.luau new file mode 100644 index 000000000..08b38543f --- /dev/null +++ b/apps/zgx/ui/ds/composer_clock.luau @@ -0,0 +1,4 @@ +-- The composer's snap/settle deadlines use the same clock in its inner +-- geometry and its page placement. Production keeps ui.now; the headless +-- motion fixture replaces this function before either composer is built. +return { now = ui.now } diff --git a/apps/zgx/ui/screens/chat/composer.luau b/apps/zgx/ui/screens/chat/composer.luau index 81b242aac..850ac2c7e 100644 --- a/apps/zgx/ui/screens/chat/composer.luau +++ b/apps/zgx/ui/screens/chat/composer.luau @@ -17,6 +17,7 @@ local ds = require("@ds") local motion = require("@ds/motion") +local clock = require("@ds/composer_clock") local T = require("@ds/transcript") local prefs = require("@app/screens/settings/prefs") local store = require("@app/screens/chat/store") @@ -1009,7 +1010,7 @@ local function Composer(): Node end if had_draft then had_draft = false - rest_settles_at = ui.now() + REST_SETTLE_MS + rest_settles_at = clock.now() + REST_SETTLE_MS end if tall <= 0 or not new_chat() then return @@ -1019,7 +1020,7 @@ local function Composer(): Node rest_tall:set(tall) end end - local left = rest_settles_at - ui.now() + local left = rest_settles_at - clock.now() if left > 0 and rest_tall:peek() ~= nil then ui.after(left, take) else @@ -1076,7 +1077,7 @@ local function Composer(): Node gliding:cancel() gliding = nil end - if motion.reduced_now() or ui.now() < snap_until or math.abs(place:peek() - y) < 0.5 then + if motion.reduced_now() or clock.now() < snap_until or math.abs(place:peek() - y) < 0.5 then place:set(y) return end @@ -1098,7 +1099,7 @@ local function Composer(): Node was_new = nc slide_next = false if switched then - snap_until = ui.now() + SNAP_MS + snap_until = clock.now() + SNAP_MS snap:set(snap:peek() + 1) end form_new:set(nc) @@ -1133,7 +1134,7 @@ local function Composer(): Node local now = route.chat:get().session if now ~= chat_was then chat_was = now - snap_until = ui.now() + SNAP_MS + snap_until = clock.now() + SNAP_MS snap:set(snap:peek() + 1) end end) @@ -1149,7 +1150,7 @@ local function Composer(): Node local now = not shell_view.collapsed:get() and fit.current:get().sidebar if now ~= sidebar_was then sidebar_was = now - snap_until = ui.now() + SNAP_MS + snap_until = clock.now() + SNAP_MS end end) -- Over the whole pane, the pointer passing through it to the page diff --git a/apps/zgx/ui/templates/zeron-v2/.base.json b/apps/zgx/ui/templates/zeron-v2/.base.json index b9f5bdd82..a22688854 100644 --- a/apps/zgx/ui/templates/zeron-v2/.base.json +++ b/apps/zgx/ui/templates/zeron-v2/.base.json @@ -1,7 +1,7 @@ { "ds/sidebar.luau": "fe41bd32634d1aba", "ds/tokens.luau": "1e102d44820a2e42", - "screens/chat/composer.luau": "d8d9cbd4198bfb90", + "screens/chat/composer.luau": "0451fa23d23c305e", "screens/chat/lane_tab.luau": "782ebff9bc222ad8", "screens/panel/init.luau": "999e58c827c6c9ff", "screens/shell/init.luau": "0e12bb5577822cf5", diff --git a/apps/zgx/ui/templates/zeron-v2/screens/chat/composer.luau b/apps/zgx/ui/templates/zeron-v2/screens/chat/composer.luau index 294ce62cf..9faf744cd 100644 --- a/apps/zgx/ui/templates/zeron-v2/screens/chat/composer.luau +++ b/apps/zgx/ui/templates/zeron-v2/screens/chat/composer.luau @@ -17,6 +17,7 @@ local ds = require("@ds") local motion = require("@ds/motion") +local clock = require("@ds/composer_clock") local T = require("@ds/transcript") local Quote = require("@ds/quote") local Comments = require("@app/screens/chat/comments") @@ -1241,7 +1242,7 @@ local function Composer(): Node end if had_draft then had_draft = false - rest_settles_at = ui.now() + REST_SETTLE_MS + rest_settles_at = clock.now() + REST_SETTLE_MS end if tall <= 0 or not new_chat() then return @@ -1251,7 +1252,7 @@ local function Composer(): Node rest_tall:set(tall) end end - local left = rest_settles_at - ui.now() + local left = rest_settles_at - clock.now() if left > 0 and rest_tall:peek() ~= nil then ui.after(left, take) else @@ -1308,7 +1309,7 @@ local function Composer(): Node gliding:cancel() gliding = nil end - if motion.reduced_now() or ui.now() < snap_until or math.abs(place:peek() - y) < 0.5 then + if motion.reduced_now() or clock.now() < snap_until or math.abs(place:peek() - y) < 0.5 then place:set(y) return end @@ -1330,7 +1331,7 @@ local function Composer(): Node was_new = nc slide_next = false if switched then - snap_until = ui.now() + SNAP_MS + snap_until = clock.now() + SNAP_MS snap:set(snap:peek() + 1) end form_new:set(nc) From d22ecffc6fed35292b491822e805fd4827ba178f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gurruchaga?= Date: Sun, 8 Nov 2026 18:32:10 -0300 Subject: [PATCH 07/81] refactor(zgx): centralize composer text metrics --- apps/zgx/docs/composer-metrics.md | 83 ++++++++ apps/zgx/src/composer_metrics_tests.rs | 221 ++++++++++++++++++++++ apps/zgx/src/main.rs | 2 + apps/zgx/ui/composer_metrics_fixture.luau | 38 ++++ apps/zgx/ui/ds/composer.luau | 59 +++--- apps/zgx/ui/ds/composer_metrics.luau | 154 +++++++++++++++ 6 files changed, 521 insertions(+), 36 deletions(-) create mode 100644 apps/zgx/docs/composer-metrics.md create mode 100644 apps/zgx/src/composer_metrics_tests.rs create mode 100644 apps/zgx/ui/composer_metrics_fixture.luau create mode 100644 apps/zgx/ui/ds/composer_metrics.luau diff --git a/apps/zgx/docs/composer-metrics.md b/apps/zgx/docs/composer-metrics.md new file mode 100644 index 000000000..7f13d0bfb --- /dev/null +++ b/apps/zgx/docs/composer-metrics.md @@ -0,0 +1,83 @@ +# Composer text metrics + +`ui/ds/composer_metrics.luau` owns the composer's two invisible text nodes +and their measurements. `composer.luau` consumes its result for compact +vs expanded layout and for the expanded text box's height. It no longer +owns `line_w`, `wrapped_h` or their resize callbacks. + +This is an adapter around the existing plain-text measurement provider, +not an editor measurement API or a fix for the reported visual glitch. +The visible editor still performs its own shaping/highlighting. Replacing +this provider with editor metrics remains separate work. + +## Contract + +`Metrics.use { value, width, left, keep? }` returns `nodes` (exactly two +text nodes, placed directly in the existing surface) and `result` (a +computed snapshot): + +| Field | Meaning | +| --- | --- | +| `result.revision` | Current requested content/style/width generation | +| `result.width` | Requested expanded content width in logical points | +| `result.status` | `pending`, `stale` or `ready` | +| `result.sample` | Complete last measurement, or nil before any measurement | +| `sample.unwrapped_width` | Natural width used to decide compact vs expanded | +| `sample.wrapped_height` | Natural height at `sample.width`, including a final empty line | +| `sample.line_height` | Resolved composer line-height token for that request | +| `sample.width` | Actual width at which the wrapped text was sampled | +| `sample.revision` | Request generation represented by the sample | + +`pending` has no sample. `stale` retains a complete old sample while new +content, style or width is being measured. `ready` identifies a sample for +the current request. A zero unwrapped width is valid for a measured empty +draft; it is never used as an initialization sentinel. Unknown/zero +available widths cannot publish a fresh sample. + +The consumer deliberately keeps the last sample while stale, preserving +the previous layout until new measures arrive. Before the first sample, +it retains the previous expansion decision and uses the existing height +floor. A newline or explicitly expanded composer still opens immediately. + +With `keep`, the complete sample survives a reload. It starts stale in the +new instance and is revalidated against new nodes; retained numbers are +not silently labelled as a measurement performed by the new instance. + +## Width and publication + +The expanded content width is calculated once at the boundary to the +adapter: available pill width minus both one-point borders and both +16-point expanded text insets. The wrapping node receives that explicit +width. The animated field's changing left/right insets never feed back +into this input. Parent/window width changes still update available space. + +Requests change with content, available width and theme typography inputs. +Both resize callbacks and changed requests schedule a coalesced deferred +read of the two nodes. This also updates revisions for equal-size edits, +which do not emit resize notifications. Callbacks for replaced requests +are discarded. One state write publishes both axes and their metadata; +independent width/height state writes cannot mix different requests. + +Each new schedule explicitly replaces the old timer. ZGX owns timers +created inside an effect until its next run: relying on a `queued` flag +alone could wait forever after that timer was automatically cancelled. +Cleanup cancels remaining work. No polling timer runs while idle. + +Publication is now coalesced, so an animation can begin in a different +frame from the old independent callbacks. Duration/easing and layout rules +are unchanged. The C1 before/after traces have identical destination +rectangles and wrap boundaries for all 786 observed samples; intermediate +animation samples are not claimed to be frame-for-frame identical. + +## Validation + +```sh +cargo test --locked --manifest-path apps/zgx/Cargo.toml --bin zeron-zgx composer +``` + +`composer_metrics_tests` covers pending vs empty, stale/replaced requests, +equal-size edits without remounting nodes, target-width independence from +the presented parent, wrapping, trailing newlines, reload and typography +changes. The C1 route-motion tests continue to cover editor identity, +focus, selection and transitions. These are headless checks, not pixel or +macOS presentation validation. diff --git a/apps/zgx/src/composer_metrics_tests.rs b/apps/zgx/src/composer_metrics_tests.rs new file mode 100644 index 000000000..e77b3be09 --- /dev/null +++ b/apps/zgx/src/composer_metrics_tests.rs @@ -0,0 +1,221 @@ +//! The measurement contract, exercised with the real invisible text nodes. +use zgx_script::{ScriptHost, Value, testing::TestApp}; + +fn mount() -> TestApp { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("ui/composer_metrics_fixture.luau"); + let app = TestApp::new(|runtime| ScriptHost::new(runtime, path)); + app.ui.scene.borrow_mut().resize(800., 600.); + app.frames(3); + assert_eq!(app.error(), None); + app +} + +fn result(app: &TestApp) -> Value { + app.host + .states() + .into_iter() + .find(|(n, _)| n == "fixture_metrics_result") + .expect("fixture result") + .1 +} + +fn number(value: &Value, key: &str) -> f64 { + value + .field(key) + .and_then(Value::as_f64) + .unwrap_or_else(|| panic!("no {key}: {value:?}")) +} + +fn ready(app: &TestApp) -> Value { + let r = result(app); + assert_eq!(r.field("status"), Some(&Value::from("ready")), "{r:?}"); + let s = r.field("sample").expect("measured sample").clone(); + assert_eq!(number(&s, "revision"), number(&r, "revision"), "{r:?}"); + assert!( + (number(&s, "width") - number(&r, "width")).abs() < 0.5, + "{r:?}" + ); + s +} + +fn two_nodes(app: &mut TestApp) -> Vec { + let nodes = [".composer-metric-line", ".composer-metric-wrapped"] + .iter() + .map(|selector| app.find(selector).expect("measurement node")) + .collect::>(); + assert_ne!(nodes[0], nodes[1]); + assert!(app.find(".composer-metric-line[2]").is_none()); + assert!(app.find(".composer-metric-wrapped[2]").is_none()); + nodes +} + +#[test] +fn composer_metrics_publish_empty_text_as_a_real_sample_then_refresh_equal_size_edits() { + let mut app = mount(); + let nodes = two_nodes(&mut app); + let history = app + .host + .states() + .into_iter() + .find(|(n, _)| n == "fixture_metrics_history") + .unwrap() + .1; + assert_eq!( + history.list().first(), + Some(&Value::from("pending")), + "initial zeroes are not a measured empty draft" + ); + let empty = ready(&app); + assert_eq!(number(&empty, "unwrapped_width"), 0., "{empty:?}"); + assert!(number(&empty, "wrapped_height") >= number(&empty, "line_height")); + let mut previous = empty; + for text in ["abc", "cba"] { + app.host + .set_state("fixture_metrics_text", Value::from(text)) + .unwrap(); + assert_eq!(result(&app).field("status"), Some(&Value::from("stale"))); + app.frames(3); + let sample = ready(&app); + assert!(number(&sample, "unwrapped_width") > 0.); + assert!(number(&sample, "revision") > number(&previous, "revision")); + if text == "cba" { + assert_eq!( + number(&sample, "unwrapped_width"), + number(&previous, "unwrapped_width") + ); + assert_eq!( + number(&sample, "wrapped_height"), + number(&previous, "wrapped_height") + ); + } + assert_eq!( + two_nodes(&mut app), + nodes, + "edits must retain the same two measuring nodes" + ); + previous = sample; + } +} + +#[test] +fn composer_metrics_reject_unknown_width_and_coalesce_replaced_requests() { + let app = mount(); + let before = ready(&app); + app.host + .set_state("fixture_metrics_width", Value::Num(0.)) + .unwrap(); + app.frames(3); + let unknown = result(&app); + assert_eq!(unknown.field("status"), Some(&Value::from("stale"))); + assert_eq!( + unknown.field("sample"), + Some(&before), + "zero width cannot replace a valid sample" + ); + for (width, text) in [ + (100., "old"), + (250., "intermediate"), + (180., "latest\nline\n"), + ] { + app.host + .set_state("fixture_metrics_width", Value::Num(width)) + .unwrap(); + app.host + .set_state("fixture_metrics_text", Value::from(text)) + .unwrap(); + } + app.frames(3); + let sample = ready(&app); + assert_eq!(number(&sample, "width"), 180.); + assert_eq!( + number(&sample, "wrapped_height"), + 3. * number(&sample, "line_height") + ); + let settled = result(&app); + app.frames(30); + assert_eq!(result(&app), settled, "idle must not publish new revisions"); +} + +#[test] +fn composer_metrics_keep_the_target_width_when_the_presented_parent_resizes() { + let app = mount(); + app.host + .set_state( + "fixture_metrics_text", + Value::from("words ".repeat(30).as_str()), + ) + .unwrap(); + app.frames(3); + let before = ready(&app); + for width in [600., 400., 200., 500.] { + app.host + .set_state("fixture_metrics_viewport", Value::Num(width)) + .unwrap(); + app.frames(2); + assert_eq!( + ready(&app), + before, + "a transient viewport width must not change target metrics" + ); + } +} + +#[test] +fn composer_metrics_revalidate_restored_samples_and_typography_changes() { + let mut app = mount(); + app.host + .set_state("fixture_metrics_text", Value::from("some words")) + .unwrap(); + app.frames(3); + let before = ready(&app); + app.host.reload().unwrap(); + assert_eq!( + result(&app).field("status"), + Some(&Value::from("stale")), + "restored measures need a fresh layout" + ); + app.frames(3); + let restored = ready(&app); + assert!(number(&restored, "revision") > number(&before, "revision")); + two_nodes(&mut app); + assert!(app.host.run_command("fixture.metrics.type")); + app.frames(3); + let changed = ready(&app); + assert_eq!(number(&changed, "line_height"), 28.); + assert!(number(&changed, "revision") > number(&restored, "revision")); + assert!(number(&changed, "unwrapped_width") > number(&restored, "unwrapped_width")); +} + +#[test] +fn composer_metrics_wrap_at_requested_width_and_preserve_a_final_empty_line() { + let app = mount(); + app.host + .set_state("fixture_metrics_text", Value::from("one\ntwo\n")) + .unwrap(); + app.frames(3); + let multiline = ready(&app); + assert!( + number(&multiline, "wrapped_height") >= 3. * number(&multiline, "line_height"), + "{multiline:?}" + ); + app.host + .set_state( + "fixture_metrics_text", + Value::from("words ".repeat(30).as_str()), + ) + .unwrap(); + app.frames(3); + let wide = ready(&app); + app.host + .set_state("fixture_metrics_width", Value::Num(120.)) + .unwrap(); + app.frames(3); + let narrow = ready(&app); + assert_eq!(number(&narrow, "width"), 120.); + assert!(number(&narrow, "wrapped_height") > number(&wide, "wrapped_height")); + assert_eq!( + number(&narrow, "unwrapped_width"), + number(&wide, "unwrapped_width") + ); +} diff --git a/apps/zgx/src/main.rs b/apps/zgx/src/main.rs index 0b28ad07c..ef0cd51e0 100644 --- a/apps/zgx/src/main.rs +++ b/apps/zgx/src/main.rs @@ -76,6 +76,8 @@ mod split_tests; #[cfg(test)] mod route_motion_tests; #[cfg(test)] +mod composer_metrics_tests; +#[cfg(test)] mod hit_sweep_tests; #[cfg(test)] mod hit_tests; diff --git a/apps/zgx/ui/composer_metrics_fixture.luau b/apps/zgx/ui/composer_metrics_fixture.luau new file mode 100644 index 000000000..d1dd3f1ed --- /dev/null +++ b/apps/zgx/ui/composer_metrics_fixture.luau @@ -0,0 +1,38 @@ +local ds = require("@ds") +local Metrics = require("@ds/composer_metrics") +ds.setup() + +local value = ui.state("fixture_metrics_text", "") +local width = ui.state("fixture_metrics_width", 320) +local viewport = ui.state("fixture_metrics_viewport", 800) +local metrics = Metrics.use { + value = value, + width = function() return width:get() end, + left = 16, + keep = "fixture", +} +local result = ui.state("fixture_metrics_result", {} :: any) +local history = ui.state("fixture_metrics_history", {} :: { string }) +ui.effect(function() + local r = metrics.result:get() + result:set(r) + local h = table.clone(history:peek()) + table.insert(h, r.status) + history:set(h) +end) + +ui.command { + id = "fixture.metrics.type", + title = "Larger composer type", + run = function() + ui.theme.tokens { zeron = { composer = { text = 18, line = 28 } } } + end, +} + +return ui.div { + w = viewport, + h_full = true, + relative = true, + metrics.nodes[1], + metrics.nodes[2], +} diff --git a/apps/zgx/ui/ds/composer.luau b/apps/zgx/ui/ds/composer.luau index f505a7df2..4801a223b 100644 --- a/apps/zgx/ui/ds/composer.luau +++ b/apps/zgx/ui/ds/composer.luau @@ -55,6 +55,7 @@ local locale = require("@self/locale") local context_usage = require("@self/context_usage") local motion = require("@self/motion") local clock = require("@ds/composer_clock") +local TextMetrics = require("@self/composer_metrics") local EditorMenu = require("@self/editor_menu") -- What the model's context holds, as its agent reports it: the tokens in @@ -953,8 +954,16 @@ return ui.component(function(props: Props): Node return if props.keep then ui.state(`composer:{props.keep}:{key}`, initial) else ui.use_state(initial) end local pill_w = kept("pill_w", 0) - local line_w = kept("line_w", 0) - local wrapped_h = kept("wrapped_h", 0) + local metrics = TextMetrics.use { + value = value, + keep = props.keep, + left = TEXT_OPEN_SIDE, + width = function(): number + -- The surface's content box excludes its two hairlines. Both + -- expanded text insets come out of that, never the eased field. + return pill_w:get() - HAIRLINES - TEXT_OPEN_SIDE * 2 + end, + } local head_h_seen = kept("head_h", 0) local trailing_seen = kept("trailing_w", 0) local footer_w = kept("footer_w", 0) @@ -1121,7 +1130,9 @@ return ui.component(function(props: Props): Node local text = value:get() local width = pill_w:get() local tail = trailing_w() - local need = line_w:get() + local measurement = metrics.result:get() + -- While a request is stale, retain its complete previous sample, + -- as before, rather than combining axes from different layouts. if width <= 0 then return false end @@ -1129,10 +1140,13 @@ return ui.component(function(props: Props): Node local want if string.find(text, "\n") then want = true + elseif measurement.sample == nil then + -- No measured text yet; zero isn't a measured empty draft. + want = was_open elseif was_open then - want = need > room - HYSTERESIS + want = measurement.sample.unwrapped_width > room - HYSTERESIS else - want = need > room + want = measurement.sample.unwrapped_width > room end was_open = want return want @@ -1148,7 +1162,8 @@ return ui.component(function(props: Props): Node -- The text box opened out: the wrapped text with its insets, between -- the floor and the cap. local box = ui.computed(function(): number - local h = math.max(wrapped_h:get(), LINE) + local sample = metrics.result:get().sample + local h = if sample then math.max(sample.wrapped_height, sample.line_height) else LINE return math.clamp(h + TEXT_TOP + TEXT_BOTTOM, TEXT_MIN, TEXT_MAX) end) local HEAD_GAP = head_gap() @@ -1788,36 +1803,8 @@ return ui.component(function(props: Props): Node end end, }, - -- The measures: the draft on one line, and wrapped as it would be - -- opened out (a mark after it, so a last empty line counts). - ui.text { - on_resize = measure(line_w, "w"), - absolute = true, - left = 0, - top = 0, - opacity = 0, - pointer_events = "none", - text_size = "$zeron.composer.text", - line_height = "$zeron.composer.line", - function() - return value:get() - end, - }, - ui.text { - on_resize = measure(wrapped_h, "h"), - absolute = true, - left = TEXT_OPEN_SIDE, - right = TEXT_OPEN_SIDE, - top = 0, - opacity = 0, - pointer_events = "none", - wrap = true, - text_size = "$zeron.composer.text", - line_height = "$zeron.composer.line", - function() - return value:get() .. "\u{200B}" - end, - }, + metrics.nodes[1], + metrics.nodes[2], head, -- The draft. ui.div { diff --git a/apps/zgx/ui/ds/composer_metrics.luau b/apps/zgx/ui/ds/composer_metrics.luau new file mode 100644 index 000000000..884745d8e --- /dev/null +++ b/apps/zgx/ui/ds/composer_metrics.luau @@ -0,0 +1,154 @@ +-- The composer's measurement provider. For now the same two invisible +-- plain-text nodes measure its unwrapped width and its expanded height. +-- Keep this contract independent of those nodes so an editor-backed +-- provider can replace them without changing layout rules. +local M = {} + +export type Sample = { + unwrapped_width: number, + wrapped_height: number, + line_height: number, + width: number, + revision: number, +} +export type Result = { + -- A complete last sample, never one axis from a different request. + sample: Sample?, + status: "pending" | "stale" | "ready", + revision: number, + width: number, +} +export type Props = { + value: State, + -- Content width at the expanded destination, not the animated viewport. + width: () -> number, + left: number, + keep: string?, +} + +function M.use(props: Props) + local measured: State = if props.keep + then ui.state(`composer:{props.keep}:text_metrics`, nil :: Sample?) + else ui.use_state(nil :: Sample?) + local revision = if measured:peek() then (measured:peek() :: Sample).revision else 0 + local request = ui.computed(function() + local text = props.value:get() + local width = math.max(0, props.width()) + local theme = ui.theme.current:get() + local size = (ui.theme.get("zeron.composer.text") :: number?) or 14 + local line = (ui.theme.get("zeron.composer.line") :: number?) or 22 + local font = ui.theme.get("font.sans") + revision += 1 + return { + text = text, + width = width, + size = size, + line = line, + font = font, + theme = theme, + revision = revision, + } + end) + local line, wrapped = ui.ref(), ui.ref() + local alive = true + local cancel: (() -> ())? = nil + local queue: () -> () + queue = function() + if not alive then + return + end + -- Timers created in an effect are cancelled when it re-runs. Always + -- replace the handle: a retained "queued" flag could otherwise leave + -- the provider waiting forever for a callback that was cancelled. + if cancel then + cancel() + end + local wanted = request:peek() + -- Coalesce both resize notifications into one read of both nodes. + -- Also sample on a request change: equal-size edits have no resize. + cancel = ui.after(0, function() + if not alive then + return + end + if request:peek().revision ~= wanted.revision then + queue() + return + end + local a, b = line:peek(), wrapped:peek() + if not a or not b or wanted.width <= 0 then + return + end + local w, h, at = a.width, b.height, b.width + if w == nil or h == nil or at == nil or h <= 0 or math.abs(at - wanted.width) >= 0.5 then + return + end + measured:set({ + unwrapped_width = w, + wrapped_height = h, + line_height = wanted.line, + width = at, + revision = wanted.revision, + }) + end) + end + ui.effect(function() + request:get() + queue() + end) + ui.on_cleanup(function() + alive = false + if cancel then + cancel() + end + end) + + local result = ui.computed(function(): Result + local want, sample = request:get(), measured:get() + return { + sample = sample, + status = if sample == nil then "pending" elseif sample.revision == want.revision then "ready" else "stale", + revision = want.revision, + width = want.width, + } + end) + local nodes = { + ui.text { + class = "composer-metric-line", + ref = line, + on_resize = queue, + absolute = true, + left = 0, + top = 0, + opacity = 0, + pointer_events = "none", + text_size = "$zeron.composer.text", + line_height = "$zeron.composer.line", + function() + return request:get().text + end, + }, + ui.text { + class = "composer-metric-wrapped", + ref = wrapped, + on_resize = queue, + absolute = true, + left = props.left, + w = function() + return request:get().width + end, + top = 0, + opacity = 0, + pointer_events = "none", + wrap = true, + text_size = "$zeron.composer.text", + line_height = "$zeron.composer.line", + -- A final empty line still contributes to the measured height. + function() + return request:get().text .. "\u{200B}" + end, + }, + } + return { result = result, nodes = nodes } +end + +return M From 59f475e3c8f37a9a4576391406d44d4fb2072e6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gurruchaga?= Date: Sun, 8 Nov 2026 18:43:39 -0300 Subject: [PATCH 08/81] refactor(zgx): compute composer geometry in one place --- apps/zgx/docs/composer-layout.md | 88 +++++++++ apps/zgx/src/composer_layout_tests.rs | 90 +++++++++ apps/zgx/src/main.rs | 2 + apps/zgx/ui/composer_fixture.luau | 4 + apps/zgx/ui/ds/composer.luau | 196 +++++--------------- apps/zgx/ui/ds/composer_layout.luau | 122 ++++++++++++ apps/zgx/ui/tests/composer_layout.test.luau | 150 +++++++++++++++ 7 files changed, 505 insertions(+), 147 deletions(-) create mode 100644 apps/zgx/docs/composer-layout.md create mode 100644 apps/zgx/src/composer_layout_tests.rs create mode 100644 apps/zgx/ui/ds/composer_layout.luau create mode 100644 apps/zgx/ui/tests/composer_layout.test.luau diff --git a/apps/zgx/docs/composer-layout.md b/apps/zgx/docs/composer-layout.md new file mode 100644 index 000000000..520c9a6aa --- /dev/null +++ b/apps/zgx/docs/composer-layout.md @@ -0,0 +1,88 @@ +# Composer target geometry + +`ui/ds/composer_layout.luau` is the pure calculation for the pill, text +viewport and control positions. It reads no UI state, theme, time or refs, +and mutates neither its input nor expansion history. It centralizes the +compact line size, border, insets, text min/max and hysteresis thresholds. + +## Inputs and outputs + +`calculate(input, previous_text_expanded)` receives available pill width, +the complete last text-metric sample, measured trailing controls/header, +resolved theme corner and header gap, and the explicit mode flags +(expanded, cargo, asking, dictating, multiline). A missing sample and an +unknown width have explicit fallback behavior. As in C2, the integration +keeps the last complete sample while a new request is stale. + +The result contains: + +- Effective `mode`/`expanded` and `next_text_expanded` for the next input. +- `natural_height`, before the text box's minimum/maximum, and `height`, + the limited target pill height including header, actions and borders. +- The text rectangle: left/right/top insets, width and viewport height. +- Leading/trailing control anchors and the dictation track's anchors. +- The tray chip inset, aligned with the same resolved control corner. + +Coordinates are local to the surface's content box. The text width excludes +the surface border and both text insets; it never supplies the width used +to measure text during an animation. `measurement(width)` shares the +expanded-width calculation with C2's provider without creating a dependency +cycle between measuring content and calculating the resulting height. +The view retains its radius, typography, voice-track height and decoration +tokens; the calculation receives resolved values where numeric geometry +depends on theme tokens. `control.nest` still resolves special pill radii. + +## Explicit hysteresis + +Text expands above its compact capacity and contracts only below the +existing smaller threshold. The pure function receives the previous text +decision and returns the next one. No computed callback mutates a captured +`was_open` variable anymore. + +The **effective mode** and **text history** are intentionally distinct. +As before, a question, cargo, dictation or forced expanded form opens the +composer without overwriting text hysteresis. Unknown width preserves the +previous text history too. Explicit multiline text does update it once +width is known. This distinction preserves behavior after forced modes end. + +`composer.luau` gathers the inputs in one place, invokes the calculation +in an effect, and commits the next history plus `target_geometry` in one +batch. A scalar selector takes each destination from this single result. +Unchanged destinations retain their values. + +## Target, presented and published sizes + +`target_geometry` describes the destination, not an intermediate frame. +The `presented` helper still applies the existing independent scalar +animations, easing, duration and snap policy. Coordinating the animations +belongs to C4; this change does not implement that policy. + +`natural_height` is not the box's displayed height and is not published to +the transcript. `screens/chat/composer.luau` continues to publish the actual +laid-out dock height through `shared.dock_h`, as before. Page placement, +footer folding and deck/tray behavior retain their own responsibilities. +The footer simply obtains its existing chip alignment from the shared +geometry result rather than repeating a control-inset calculation. + +## Validation + +```sh +cargo test --locked --manifest-path apps/zgx/Cargo.toml --bin zeron-zgx composer +``` + +`ui/tests/composer_layout.test.luau` tests eight cases through the Rust +test runner: golden compact geometry, both exact hysteresis boundaries, +forced modes/history, missing measurements, natural vs limited height, +cargo/questions with headers, theme corners, and deterministic calculation +against frozen inputs. Expected sizes are examples and invariants, rather +than a second implementation of the layout formulas. + +An integration test adds/removes a real cargo header and question, checking +editor identity, focus, selection, draft retention, header/text separation +and return to compact geometry. C1/C2 characterization and metric tests +remain part of the same targeted suite. These are headless checks; they do +not establish that the original visual glitch is fixed. + +The C1 before/after trace comparison against C2 covered 786 samples. After +normalizing editor node IDs, all observed values matched, including target +rectangles and intermediate presented geometry, focus, selection and draft. diff --git a/apps/zgx/src/composer_layout_tests.rs b/apps/zgx/src/composer_layout_tests.rs new file mode 100644 index 000000000..0c52cf998 --- /dev/null +++ b/apps/zgx/src/composer_layout_tests.rs @@ -0,0 +1,90 @@ +//! Pure geometry rules run in Luau, where the production calculation lives. +use zgx_script::{ScriptHost, Value, testing::TestApp}; + +#[test] +fn composer_geometry_rules_have_boundary_header_and_purity_coverage() { + let ui = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("ui"); + let report = zgx_script::test_runner::run(&ui, Some("composer geometry:"), |outcome| { + if let Some(failure) = &outcome.failure { + eprintln!("{}: {failure}", outcome.name); + } + }) + .expect("geometry tests load"); + assert_eq!(report.outcomes.len(), 8, "all geometry cases must run"); + assert_eq!(report.failed(), 0, "{report:?}"); +} + +#[test] +fn composer_headers_grow_and_shrink_without_replacing_or_covering_the_editor() { + let path = + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("ui/composer_fixture.luau"); + let mut app = TestApp::new(|runtime| ScriptHost::new(runtime, path)); + app.ui.scene.borrow_mut().resize(700., 600.); + app.frames(30); + app.fill("role:textbox", "keep this draft"); + app.frames(30); + let field = app.find("role:textbox").unwrap(); + let editor = app.ui.editor_within(field).unwrap(); + editor.set_selection(1, 8); + let short = app.bounds("role:textbox"); + for (state, value, header) in [ + ( + "test_composer_cargo", + r#"[{"id":"f1","kind":"file","name":"notes.md"}]"#, + "notes.md", + ), + ( + "test_composer_question", + r#"{"text":"Choose a branch","options":[{"value":"main","label":"Main"}]}"#, + "Choose a branch", + ), + ] { + app.host + .set_state(state, Value::from_json(value).unwrap()) + .unwrap(); + for _ in 0..30 { + app.frames(1); + let current = app.find("role:textbox").unwrap(); + let handle = app.ui.editor_within(current).unwrap(); + assert_eq!(handle.node, editor.node); + assert_eq!(handle.selection(), (1, 8)); + assert_eq!(app.ui.input.focused(), Some(editor.node)); + assert_eq!(handle.editor.borrow().text().as_ref(), "keep this draft"); + assert_eq!(app.error(), None); + } + let field_box = app.bounds("role:textbox"); + let role = if state.ends_with("cargo") { + "button" + } else { + "radiogroup" + }; + let head = app + .find(&format!("role:{role} \"{header}\"")) + .unwrap_or_else(|| panic!("{state}: missing {header}\n{}", app.tree())); + let head_box = app.ui.scene.borrow().transformed_bounds(head); + assert!( + field_box.height > short.height, + "header opens the expanded editor" + ); + assert!( + head_box.y + head_box.height <= field_box.y, + "header overlaps the text: {head_box:?}, {field_box:?}" + ); + app.host + .set_state( + state, + if state.ends_with("cargo") { + Value::List(Vec::new().into()) + } else { + Value::Nil + }, + ) + .unwrap(); + app.frames(40); + let back = app.bounds("role:textbox"); + assert!( + (back.y - short.y).abs() < 1. && (back.height - short.height).abs() < 1., + "removing header restores the compact draft" + ); + } +} diff --git a/apps/zgx/src/main.rs b/apps/zgx/src/main.rs index ef0cd51e0..fbf014057 100644 --- a/apps/zgx/src/main.rs +++ b/apps/zgx/src/main.rs @@ -78,6 +78,8 @@ mod route_motion_tests; #[cfg(test)] mod composer_metrics_tests; #[cfg(test)] +mod composer_layout_tests; +#[cfg(test)] mod hit_sweep_tests; #[cfg(test)] mod hit_tests; diff --git a/apps/zgx/ui/composer_fixture.luau b/apps/zgx/ui/composer_fixture.luau index 351e4a07f..56fd7a3cc 100644 --- a/apps/zgx/ui/composer_fixture.luau +++ b/apps/zgx/ui/composer_fixture.luau @@ -6,6 +6,8 @@ ui.theme.strict("warn") local ds = require("@ds") local Composer = require("@ds/composer") +local Cargo = require("@ds/composer_cargo") +local Ask = require("@ds/composer_ask") ds.setup() local HARNESSES: { ds.Harness } = { { id = "claude", label = "Claude", mark = "claude" } } @@ -28,6 +30,8 @@ return ui.column { harnesses = HARNESSES, models = MODELS, notice = ui.state("test_composer_notice", nil :: Composer.Notice?), + cargo = ui.state("test_composer_cargo", {} :: { Cargo.Item }), + question = ui.state("test_composer_question", nil :: Ask.Question?), project = ui.state("test_composer_project", "atlas" :: string?), projects = PROJECTS, }, diff --git a/apps/zgx/ui/ds/composer.luau b/apps/zgx/ui/ds/composer.luau index 4801a223b..72a8402bb 100644 --- a/apps/zgx/ui/ds/composer.luau +++ b/apps/zgx/ui/ds/composer.luau @@ -56,6 +56,7 @@ local context_usage = require("@self/context_usage") local motion = require("@self/motion") local clock = require("@ds/composer_clock") local TextMetrics = require("@self/composer_metrics") +local Layout = require("@self/composer_layout") local EditorMenu = require("@self/editor_menu") -- What the model's context holds, as its agent reports it: the tokens in @@ -274,39 +275,6 @@ export type Props = { mention_empty: { [string]: string }?, } --- Zeron's geometry. The one line: 47 points inside the hairline, the text --- centred in it. Opened out: the text box (16 above the text, 4 below) --- between its floor and cap, then the controls' row (2, a 32-point chip --- zone, 8). Round controls are 28 across, concentric with the pill's --- corner: on one line (a pill, its corner half its height) centred in it, --- as far from its sides as from its foot; opened out, as far in from the --- side and the foot as the pill's corner (inside its hairline) is round --- less theirs. -local LINE = 22 -local ONE_LINE = 47 -local TEXT_TOP, TEXT_BOTTOM, TEXT_MIN, TEXT_MAX = 16, 4, 76, 260 -local ACTIONS = 42 -local HAIRLINES = 2 -local CONTROL = 28 -local INSET_LINE = (ONE_LINE - CONTROL) / 2 -local function open_inset(): number - return control.nest("$zeron.composer.radius", 1) - CONTROL / 2 -end --- The controls' row opened out: its 42 points, or more where a rounder --- pill (a template's) sets its controls further in from the foot, so they --- keep the room over them they have at Zeron's own corner (its controls' --- top 3 under the row's) rather than rising into the text. -local ACTIONS_OVER = 3 -local function actions(): number - return math.max(ACTIONS, open_inset() + CONTROL + ACTIONS_OVER) -end --- The draft's own inset either side on one line, and opened out. -local TEXT_SIDE = 8 -local TEXT_OPEN_SIDE = 16 --- Opened out, the pill folds back to one line only once the text is this --- much narrower than the line holds, so a width at the boundary cannot --- flicker between the two. -local HYSTERESIS = 32 -- How wide where-it-works stays before the access mode folds: its project's -- name (to PROJECT_MAX) and BRANCH_KEEP of the branch's, with the chip's -- own room (its padding, the glyph, the gaps, the chevron); until @@ -922,14 +890,6 @@ fn fill(input: FillInput) -> vec4 { local lightboxes = 0 return ui.component(function(props: Props): Node - -- Opened out, the controls' inset: from the theme's corner, as this one - -- is built. - local INSET_OPEN = open_inset() - -- On the tray, the chips' inset from the pill's edge above them, the - -- tray's foot and its sides: the pill's own controls' (+ and Send sit - -- this far in from its edge, its hairline and INSET_OPEN), so the row - -- keeps the pill's spacing and its chips line up with those buttons. - local TRAY_INSET = INSET_OPEN + HAIRLINES / 2 -- On the tray, the step under the pill and above the tray's foot (one -- token: the footer's margins and the tray's height both read it), and -- how far the tray tucks up behind the pill, its top hidden there. @@ -957,11 +917,11 @@ return ui.component(function(props: Props): Node local metrics = TextMetrics.use { value = value, keep = props.keep, - left = TEXT_OPEN_SIDE, + left = Layout.measurement(0).left, width = function(): number -- The surface's content box excludes its two hairlines. Both -- expanded text insets come out of that, never the eased field. - return pill_w:get() - HAIRLINES - TEXT_OPEN_SIDE * 2 + return Layout.measurement(pill_w:get()).width end, } local head_h_seen = kept("head_h", 0) @@ -1107,92 +1067,33 @@ return ui.component(function(props: Props): Node return (if w > 0 then w else 1000) < NARROW end - -- One line or opened out. On one line the text has the pill less the - -- leading control and the trailing ones; opened out, it folds back only - -- once comfortably short again. Anything over the draft opens it out. - -- The controls at the right, as measured; until they are (a composer - -- just made, a reload's), their usual two (the microphone and Send), - -- so the text is laid out where it rests from the first frame rather - -- than easing in from the full width. - local function trailing_w(): number - local w = trailing_seen:get() - return if w > 0 then w else CONTROL * 2 + 8 + -- A pure calculation owns all target geometry. The effect commits the + -- explicit text-hysteresis state; rendering/computeds mutate no history. + local previous_text_expanded = ui.use_state(read(props.expanded) == true) + local function layout_input(): Layout.Input + local _ = ui.theme.current:get() + return { + width = pill_w:get(), + -- Retain C2's complete previous sample while a request is stale. + sample = metrics.result:get().sample, + trailing_width = trailing_seen:get(), + control_corner = control.nest("$zeron.composer.radius", Layout.BORDER), + header_height = head_h_seen:get(), + header_gap = head_gap(), + expanded = read(props.expanded) == true, + cargo = has_cargo(), + asking = asking(), + dictating = dictating(), + multiline = string.find(value:get(), "\n") ~= nil, + } end - local function expanded(): boolean - return read(props.expanded) == true - end - local was_open = expanded() - local open = ui.computed(function(): boolean - -- Dictating, the words land in the draft over the track. - if expanded() or has_cargo() or asking() or dictating() then - return true - end - local text = value:get() - local width = pill_w:get() - local tail = trailing_w() - local measurement = metrics.result:get() - -- While a request is stale, retain its complete previous sample, - -- as before, rather than combining axes from different layouts. - if width <= 0 then - return false - end - local room = width - HAIRLINES - (INSET_LINE + CONTROL + TEXT_SIDE) - (TEXT_SIDE + tail + INSET_LINE) - local want - if string.find(text, "\n") then - want = true - elseif measurement.sample == nil then - -- No measured text yet; zero isn't a measured empty draft. - want = was_open - elseif was_open then - want = measurement.sample.unwrapped_width > room - HYSTERESIS - else - want = measurement.sample.unwrapped_width > room - end - was_open = want - return want - end) - - -- What is over the draft: nothing, or its measured height. - local head_h = ui.computed(function(): number - if not has_cargo() and not asking() then - return 0 - end - return head_h_seen:get() - end) - -- The text box opened out: the wrapped text with its insets, between - -- the floor and the cap. - local box = ui.computed(function(): number - local sample = metrics.result:get().sample - local h = if sample then math.max(sample.wrapped_height, sample.line_height) else LINE - return math.clamp(h + TEXT_TOP + TEXT_BOTTOM, TEXT_MIN, TEXT_MAX) - end) - local HEAD_GAP = head_gap() - local height = ui.computed(function(): number - if open:get() then - local head = head_h:get() - -- With something over it, the draft's box starts under that - -- with the strip's own gap rather than the pill's top. - return head + (if head > 0 then box:get() - TEXT_TOP + HEAD_GAP else box:get()) + actions() + HAIRLINES - end - return ONE_LINE + HAIRLINES - end) - -- Where the text sits: beside the controls on one line, over them - -- opened out, under whatever is over it. - local text_left = ui.computed(function(): number - return if open:get() then TEXT_OPEN_SIDE else INSET_LINE + CONTROL + TEXT_SIDE - end) - local text_right = ui.computed(function(): number - return if open:get() then TEXT_OPEN_SIDE else TEXT_SIDE + trailing_w() + INSET_LINE - end) - local text_top = ui.computed(function(): number - if not open:get() then - return (ONE_LINE - LINE) / 2 - end - local head = head_h:get() - return if head > 0 then head + HEAD_GAP else TEXT_TOP - end) - local text_h = ui.computed(function(): number - return if open:get() then box:get() - TEXT_TOP - TEXT_BOTTOM else LINE + local target_geometry = ui.use_state(Layout.calculate(layout_input(), previous_text_expanded:peek())) + ui.effect(function() + local next = Layout.calculate(layout_input(), previous_text_expanded:peek()) + ui.batch(function() + previous_text_expanded:set(next.next_text_expanded) + target_geometry:set(next) + end) end) local EASE: TweenOptions = { duration = 220, ease = "out(3)" } -- Each of the pill's measures as it shows: eased (EASE) as it grows or @@ -1230,10 +1131,11 @@ return ui.component(function(props: Props): Node end) return value end - -- (One on a line, another opened out.) - local function by_open(line: number, opened: number): State + -- C3 centralizes destinations only. Each scalar still has the existing + -- independent animation; coordinating those is the next change (C4). + local function presented(select: (Layout.Geometry) -> number): State return eased(ui.computed(function(): number - return if open:get() then opened else line + return select(target_geometry:get()) end)) end @@ -1597,9 +1499,9 @@ return ui.component(function(props: Props): Node return ui.div { absolute = true, -- (Insets take points: the control, the sides, and a breath.) - left = by_open(INSET_LINE + CONTROL + 8, INSET_OPEN + CONTROL + 8), - right = by_open(INSET_LINE + CONTROL * 2 + 8 + 8, INSET_OPEN + CONTROL * 2 + 8 + 8), - bottom = by_open(INSET_LINE, INSET_OPEN), + left = presented(function(g) return g.controls.voice.left end), + right = presented(function(g) return g.controls.voice.right end), + bottom = presented(function(g) return g.controls.voice.bottom end), h = "$zeron.composer.voice.h", ref = track_w, overflow = "hidden", @@ -1764,14 +1666,14 @@ return ui.component(function(props: Props): Node on_resize = measure(pill_w, "w"), relative = true, w_full = true, - h = eased(height), + h = presented(function(g) return g.height end), overflow = "hidden", -- (Its glass, edge and shadow are the frame's, round it and the deck -- over it: one surface. Its own corner only clips what's in it, and -- its edge, clear, keeps the room the frame's takes, so everything in -- it sits where it always has.) rounded = "$zeron.composer.radius", - border = 1, + border = Layout.BORDER, border_color = "#00000000", -- (Neither `on_files` nor its overlay: files are what's around it to -- take, the drop going on up to it.) @@ -1809,10 +1711,10 @@ return ui.component(function(props: Props): Node -- The draft. ui.div { absolute = true, - left = eased(text_left), - right = eased(text_right), - top = eased(text_top), - h = eased(text_h), + left = presented(function(g) return g.text.left end), + right = presented(function(g) return g.text.right end), + top = presented(function(g) return g.text.top end), + h = presented(function(g) return g.text.height end), -- Its placeholder, the composer's own (it changes with what is -- asked): over the empty draft, taking no pointer. ui.show( @@ -1961,8 +1863,8 @@ return ui.component(function(props: Props): Node -- The controls, on the pill's foot. ui.div { absolute = true, - left = by_open(INSET_LINE, INSET_OPEN), - bottom = by_open(INSET_LINE, INSET_OPEN), + left = presented(function(g) return g.controls.leading.left end), + bottom = presented(function(g) return g.controls.leading.bottom end), ui.show(function() return not asking() end, attach), @@ -1971,10 +1873,10 @@ return ui.component(function(props: Props): Node ui.row { on_resize = measure(trailing_seen, "w"), absolute = true, - right = by_open(INSET_LINE, INSET_OPEN), + right = presented(function(g) return g.controls.trailing.right end), -- (Its row is 32: the control centred in it.) - bottom = by_open(INSET_LINE - 2, INSET_OPEN - 2), - h = 32, + bottom = presented(function(g) return g.controls.trailing.bottom end), + h = function() return target_geometry:get().controls.trailing.height end, items = "center", gap = "$space.2", ui.show(asking, ask_controls()), @@ -2204,7 +2106,7 @@ return ui.component(function(props: Props): Node return if on_tray() then `${TRAY_GAP}` else 0 end :: any, px = function(): any - return if on_tray() then ui.raw(TRAY_INSET, "the pill's controls' inset") else "$zeron.space.snug" + return if on_tray() then ui.raw(target_geometry:get().tray_inset, "the pill's controls' inset") else "$zeron.space.snug" end :: any, items = "center", gap = "$space.1", @@ -2326,7 +2228,7 @@ return ui.component(function(props: Props): Node -- footer, tucked behind the pill (its top hidden under it, its sides -- the pill's), so the chips sit on the composer's surface rather than -- on the art. They sit as far in from its sides as the pill's own - -- controls sit in it (TRAY_INSET), a tight step (TRAY_GAP) below the + -- controls sit in it, a tight step (TRAY_GAP) below the -- pill and above its foot; its height is theirs (tray_h), its corner -- the trays'. ui.presence(function(): string? diff --git a/apps/zgx/ui/ds/composer_layout.luau b/apps/zgx/ui/ds/composer_layout.luau new file mode 100644 index 000000000..80a0ffeba --- /dev/null +++ b/apps/zgx/ui/ds/composer_layout.luau @@ -0,0 +1,122 @@ +-- Pure target geometry. No theme reads, signals, animation or history +-- mutations: callers resolve theme tokens and explicitly carry the text +-- expansion decision from one input change to the next. +local M = {} + +local LINE, ONE_LINE = 22, 47 +local TEXT_TOP, TEXT_BOTTOM, TEXT_MIN, TEXT_MAX = 16, 4, 76, 260 +local CONTROL, TEXT_SIDE, TEXT_OPEN_SIDE = 28, 8, 16 +local INSET_LINE = (ONE_LINE - CONTROL) / 2 +local ACTIONS, ACTIONS_OVER, HYSTERESIS = 42, 3, 32 +M.BORDER = 1 + +export type Sample = { unwrapped_width: number, wrapped_height: number, line_height: number } +export type Input = { + width: number, + sample: Sample?, + trailing_width: number, + -- Already resolved by control.nest(radius token, border), preserving + -- control.nest's special handling of fully rounded corners. + control_corner: number, + header_height: number, + header_gap: number, + expanded: boolean, + cargo: boolean, + asking: boolean, + dictating: boolean, + multiline: boolean, +} +export type Geometry = { + mode: "compact" | "expanded", + expanded: boolean, + next_text_expanded: boolean, + natural_height: number, + height: number, + header_height: number, + tray_inset: number, + text: { left: number, right: number, width: number, top: number, height: number }, + controls: { + leading: { left: number, bottom: number }, + trailing: { right: number, bottom: number, height: number }, + voice: { left: number, right: number, bottom: number }, + }, +} + +function M.measurement(width: number) + return { left = TEXT_OPEN_SIDE, width = math.max(0, width - M.BORDER * 2 - TEXT_OPEN_SIDE * 2) } +end + +-- Also used by the footer/tray to line up with the pill's controls; their +-- own layout stays outside the text geometry calculation. +local function chrome_of(control_corner: number) + local inset = control_corner - CONTROL / 2 + return { + inset = inset, + tray_inset = inset + M.BORDER, + actions_height = math.max(ACTIONS, inset + CONTROL + ACTIONS_OVER), + } +end + +function M.calculate(p: Input, previous_text_expanded: boolean): Geometry + local chrome = chrome_of(p.control_corner) + local tail = if p.trailing_width > 0 then p.trailing_width else CONTROL * 2 + TEXT_SIDE + local line_left = INSET_LINE + CONTROL + TEXT_SIDE + local line_right = TEXT_SIDE + tail + INSET_LINE + local room = p.width - M.BORDER * 2 - line_left - line_right + local forced = p.expanded or p.cargo or p.asking or p.dictating + local next_text_expanded = previous_text_expanded + local expanded = forced + if not forced and p.width > 0 then + if p.multiline then + expanded = true + elseif p.sample == nil then + expanded = previous_text_expanded + elseif previous_text_expanded then + expanded = p.sample.unwrapped_width > room - HYSTERESIS + else + expanded = p.sample.unwrapped_width > room + end + next_text_expanded = expanded + end + -- Forced modes and an unknown width don't change text hysteresis. + -- This preserves the pre-refactor behavior after a question/dictation. + local head = if p.cargo or p.asking then math.max(0, p.header_height) else 0 + local natural_text = if p.sample then math.max(p.sample.wrapped_height, p.sample.line_height) else LINE + local natural_box = natural_text + TEXT_TOP + TEXT_BOTTOM + local box = math.clamp(natural_box, TEXT_MIN, TEXT_MAX) + local head_adjustment = if head > 0 then head + p.header_gap - TEXT_TOP else 0 + local height = if expanded then box + head_adjustment + chrome.actions_height + M.BORDER * 2 else ONE_LINE + M.BORDER * 2 + local natural_height = if expanded then natural_box + head_adjustment + chrome.actions_height + M.BORDER * 2 else height + local left = if expanded then TEXT_OPEN_SIDE else line_left + local right = if expanded then TEXT_OPEN_SIDE else line_right + local inset = if expanded then chrome.inset else INSET_LINE + return { + mode = if expanded then "expanded" else "compact", + expanded = expanded, + next_text_expanded = next_text_expanded, + -- Natural is before the text box's min/max; height is the limited + -- target pill height. Neither is its animated/presented height. + natural_height = natural_height, + height = height, + header_height = head, + tray_inset = chrome.tray_inset, + text = { + left = left, + right = right, + width = math.max(0, p.width - M.BORDER * 2 - left - right), + top = if expanded then (if head > 0 then head + p.header_gap else TEXT_TOP) else (ONE_LINE - LINE) / 2, + height = if expanded then box - TEXT_TOP - TEXT_BOTTOM else LINE, + }, + controls = { + leading = { left = inset, bottom = inset }, + trailing = { right = inset, bottom = inset - 2, height = 32 }, + voice = { + left = inset + CONTROL + TEXT_SIDE, + right = inset + CONTROL * 2 + TEXT_SIDE * 2, + bottom = inset, + }, + }, + } +end + +return M diff --git a/apps/zgx/ui/tests/composer_layout.test.luau b/apps/zgx/ui/tests/composer_layout.test.luau new file mode 100644 index 000000000..d61742e56 --- /dev/null +++ b/apps/zgx/ui/tests/composer_layout.test.luau @@ -0,0 +1,150 @@ +local Layout = require("@ds/composer_layout") + +local function input(overrides: { [string]: any }?): Layout.Input + local p = { + width = 660, + sample = { unwrapped_width = 0, wrapped_height = 22, line_height = 22 }, + trailing_width = 64, + control_corner = 25, + header_height = 0, + header_gap = 12, + expanded = false, + cargo = false, + asking = false, + dictating = false, + multiline = false, + } :: any + for k, v in overrides or {} do + p[k] = v + end + return p +end + +test("composer geometry: compact destination and measured controls", function() + local g = Layout.calculate(input(), false) + expect(g.mode).to_be("compact") + expect(g.height).to_be(49) + expect(g.natural_height).to_be(49) + expect(g.text).to_equal({ left = 45.5, right = 81.5, top = 12.5, width = 531, height = 22 }) + expect(g.controls.leading).to_equal({ left = 9.5, bottom = 9.5 }) + expect(g.controls.trailing).to_equal({ right = 9.5, bottom = 7.5, height = 32 }) + local more_controls = Layout.calculate(input({ trailing_width = 100 }), false) + expect(more_controls.text.width).to_be(495) + expect(more_controls.text.left).to_be(g.text.left) + local unknown_controls = Layout.calculate(input({ trailing_width = 0 }), false) + expect(unknown_controls).to_equal(g) +end) + +test("composer geometry: expansion and contraction have distinct exact boundaries", function() + local p = input() + p.sample = { unwrapped_width = 531, wrapped_height = 22, line_height = 22 } + expect(Layout.calculate(p, false).expanded).to_be(false) + p.sample.unwrapped_width = 531.01 + local opened = Layout.calculate(p, false) + expect(opened.expanded).to_be(true) + expect(opened.next_text_expanded).to_be(true) + p.sample.unwrapped_width = 500 + expect(Layout.calculate(p, opened.next_text_expanded).expanded).to_be(true) + expect(Layout.calculate(p, false).expanded).to_be(false) + p.sample.unwrapped_width = 499 + local closed = Layout.calculate(p, true) + expect(closed.expanded).to_be(false) + expect(closed.next_text_expanded).to_be(false) +end) + +test("composer geometry: forced modes preserve explicit text history", function() + for _, flag in { "expanded", "cargo", "asking", "dictating" } do + local p = input({ [flag] = true }) + local g = Layout.calculate(p, false) + expect(g.expanded).to_be(true) + expect(g.next_text_expanded).to_be(false) + ;(p :: any)[flag] = false + expect(Layout.calculate(p, g.next_text_expanded).mode).to_be("compact") + end + local lines = Layout.calculate(input({ multiline = true }), false) + expect(lines.expanded).to_be(true) + expect(lines.next_text_expanded).to_be(true) +end) + +test("composer geometry: missing measurements and unknown width are explicit", function() + local p = input() + p.sample = nil + expect(Layout.calculate(p, false).mode).to_be("compact") + expect(Layout.calculate(p, true).mode).to_be("expanded") + p.width = 0 + local unknown = Layout.calculate(p, true) + expect(unknown.mode).to_be("compact") + expect(unknown.next_text_expanded).to_be(true) + expect(unknown.text.width).to_be(0) + p.expanded = true + expect(Layout.calculate(p, false).mode).to_be("expanded") + expect(Layout.measurement(0)).to_equal({ left = 16, width = 0 }) + expect(Layout.measurement(660)).to_equal({ left = 16, width = 626 }) +end) + +test("composer geometry: natural height is distinct from minimum and maximum", function() + local p = input({ expanded = true }) + local small = Layout.calculate(p, false) + expect(small.height).to_be(120) + expect(small.natural_height < small.height).to_be(true) + expect(small.text.height).to_be(56) + p.sample = { unwrapped_width = 40, wrapped_height = 500, line_height = 22 } + local capped = Layout.calculate(p, false) + expect(capped.height).to_be(304) + expect(capped.text.height).to_be(240) + expect(capped.natural_height > capped.height).to_be(true) + p.sample.wrapped_height = 700 + local more = Layout.calculate(p, false) + expect(more.height).to_be(capped.height) + expect(more.text).to_equal(capped.text) + expect(more.natural_height > capped.natural_height).to_be(true) +end) + +test("composer geometry: cargo and questions place text below the measured header", function() + for _, flag in { "cargo", "asking" } do + local p = input({ [flag] = true, header_height = 90 }) + local g = Layout.calculate(p, false) + expect(g.header_height).to_be(90) + expect(g.text.top).to_be(102) + expect(g.height).to_be(206) + expect(g.text.height).to_be(56) + expect(g.controls.leading.bottom).to_be(11) + p.header_height = 30 + local smaller = Layout.calculate(p, false) + expect(smaller.text.top).to_be(42) + expect(smaller.height).to_be(146) + p.header_gap = 20 + local spaced = Layout.calculate(p, false) + expect(spaced.text.top - smaller.text.top).to_be(8) + expect(spaced.height - smaller.height).to_be(8) + end + local stale = Layout.calculate(input({ expanded = true, header_height = 90 }), false) + expect(stale.header_height).to_be(0) + expect(stale.text.top).to_be(16) + expect(stale.height).to_be(120) +end) + +test("composer geometry: resolved theme corners keep controls and action space aligned", function() + local normal = Layout.calculate(input({ expanded = true }), false) + local rounder = Layout.calculate(input({ expanded = true, control_corner = 35 }), false) + expect(rounder.controls.leading.bottom).to_be(21) + expect(rounder.controls.trailing.bottom).to_be(19) + expect(rounder.controls.voice.bottom).to_be(21) + expect(rounder.height).to_be(130) + expect(rounder.text).to_equal(normal.text) + expect(rounder.tray_inset).to_be(22) +end) + +test("composer geometry: calculation is deterministic and leaves inputs untouched", function() + local p = input({ cargo = true, header_height = 90 }) + table.freeze(p.sample :: any) + table.freeze(p) + local a = Layout.calculate(p, false) + local b = Layout.calculate(p, false) + expect(a).to_equal(b) + expect(p.cargo).to_be(true) + expect(p.header_height).to_be(90) + -- Mutating a returned rectangle cannot change a later calculation. + a.text.top = -100 + expect(Layout.calculate(p, false)).to_equal(b) +end) From 7b1ca7a43567722921d49302520f171a1fca6c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gurruchaga?= Date: Sun, 8 Nov 2026 19:09:25 -0300 Subject: [PATCH 09/81] refactor(zgx): coordinate composer geometry transitions --- apps/zgx/docs/composer-characterization.md | 10 +- apps/zgx/docs/composer-layout.md | 13 +- apps/zgx/docs/composer-motion.md | 106 +++++++++++ apps/zgx/src/composer_motion_tests.rs | 174 +++++++++++++++++++ apps/zgx/src/main.rs | 2 + apps/zgx/src/route_motion_tests.rs | 87 ++++++++++ apps/zgx/ui/composer_transition_fixture.luau | 39 +++++ apps/zgx/ui/composer_transition_preview.luau | 51 ++++++ apps/zgx/ui/ds/composer.luau | 63 ++----- apps/zgx/ui/ds/composer_layout.luau | 13 +- apps/zgx/ui/ds/composer_motion.luau | 168 ++++++++++++++++++ apps/zgx/ui/tests/composer_layout.test.luau | 2 +- apps/zgx/ui/tests/composer_motion.test.luau | 74 ++++++++ 13 files changed, 743 insertions(+), 59 deletions(-) create mode 100644 apps/zgx/docs/composer-motion.md create mode 100644 apps/zgx/src/composer_motion_tests.rs create mode 100644 apps/zgx/ui/composer_transition_fixture.luau create mode 100644 apps/zgx/ui/composer_transition_preview.luau create mode 100644 apps/zgx/ui/ds/composer_motion.luau create mode 100644 apps/zgx/ui/tests/composer_motion.test.luau diff --git a/apps/zgx/docs/composer-characterization.md b/apps/zgx/docs/composer-characterization.md index 4e4f26d35..2861f5e89 100644 --- a/apps/zgx/docs/composer-characterization.md +++ b/apps/zgx/docs/composer-characterization.md @@ -58,10 +58,12 @@ prove that the GPU painted the text or that a presented frame was correct. ## Visual reproduction procedure -Status at implementation: the original visual glitch has not been -reproduced by these headless tests. A valid standalone visual comparison -and the macOS comparison have not been executed in this change. The following is a repeatable diagnostic -procedure, not a claim that these steps reproduce it on every machine. +Status at C1: the original visual glitch was not reproduced by these +headless tests, and visual comparisons were not executed in that change. +C4 adds a host-isolated composer preview and a Linux recording described +in [composer-motion.md](composer-motion.md); macOS comparison remains pending. +The following is a repeatable diagnostic procedure, not a claim that these +steps reproduce the glitch on every machine. macOS native-caret/Metal comparison requires a separate run on macOS. Use the standalone **ZGX runner built from the revision pinned in diff --git a/apps/zgx/docs/composer-layout.md b/apps/zgx/docs/composer-layout.md index 520c9a6aa..bfe1765a7 100644 --- a/apps/zgx/docs/composer-layout.md +++ b/apps/zgx/docs/composer-layout.md @@ -47,15 +47,16 @@ width is known. This distinction preserves behavior after forced modes end. `composer.luau` gathers the inputs in one place, invokes the calculation in an effect, and commits the next history plus `target_geometry` in one -batch. A scalar selector takes each destination from this single result. -Unchanged destinations retain their values. +batch. The motion controller consumes this single destination. View bindings +read a single presented-frame snapshot rather than animating each scalar. ## Target, presented and published sizes `target_geometry` describes the destination, not an intermediate frame. -The `presented` helper still applies the existing independent scalar -animations, easing, duration and snap policy. Coordinating the animations -belongs to C4; this change does not implement that policy. +`composer_motion.luau` (C4) owns origin, destination and a single progress +animation. The `presented` helper only selects values from that controller's +frame; it starts no animation. See [composer-motion.md](composer-motion.md) +for viewport constraints, discrete mode changes and interruption behavior. `natural_height` is not the box's displayed height and is not published to the transcript. `screens/chat/composer.luau` continues to publish the actual @@ -83,6 +84,6 @@ and return to compact geometry. C1/C2 characterization and metric tests remain part of the same targeted suite. These are headless checks; they do not establish that the original visual glitch is fixed. -The C1 before/after trace comparison against C2 covered 786 samples. After +The C3 before/after trace comparison against C2 covered 786 samples. After normalizing editor node IDs, all observed values matched, including target rectangles and intermediate presented geometry, focus, selection and draft. diff --git a/apps/zgx/docs/composer-motion.md b/apps/zgx/docs/composer-motion.md new file mode 100644 index 000000000..7795afe13 --- /dev/null +++ b/apps/zgx/docs/composer-motion.md @@ -0,0 +1,106 @@ +# Coordinated composer transitions + +`ui/ds/composer_motion.luau` owns one origin, destination and progress +animation per composer. All view bindings read its complete `frame` +snapshot. Selecting height, text or control positions starts no additional +animation. The duration remains 220 ms with `out(3)` easing. + +## Retargeting and lifecycle + +A new visual destination captures the currently presented snapshot as its +origin, cancels the old clip, and starts a single new progress animation. +It never starts from an unfinished transition's old destination. Reversing +direction before any frame, when already at the new destination, settles +without creating an unnecessary animation. + +Changes to natural content height above the viewport cap, text-hysteresis +metadata, or auxiliary constraints do not restart a transition when the +visual destination is unchanged. The destination metadata is still +updated; the same progress continues with the latest constraints. + +Reduce motion is tracked, including changes during a run. It cancels the +clip and snaps all geometry together. A changed `props.snap` does the same +for context switches. The existing 400 ms context-settle window remains +so measurements arriving just after a switch also snap; replacing this +deadline is C6's work. + +Cleanup cancels the clip and marks the controller inactive. Completion +callbacks check their generation and whether the controller is alive, +so an obsolete callback cannot revive a replaced or unmounted run. + +## Width and viewport policy + +Interpolating the editor's left/right insets would change its wrapping +width on every frame. This controller instead uses two discrete widths +for a mode change at a fixed surface/control width: + +1. On expansion, retain compact width until the vertical envelope holds + a complete line and its bottom has cleared both control groups. +2. Switch to expanded width once. Position and height continue on the + same vertical trajectory; the width switch does not also jump them. +3. On contraction, take compact width immediately and keep animating + the vertical envelope. Compact controls stay beside the text. + +Snapping to full expanded width immediately at compact height would put +text under the buttons. Conversely, moving the text to its final top only +at the width switch produced a vertical jump during the initial visual +check. The final policy animates the vertical envelope throughout and +delays only the width switch until it is safe. + +The viewport is derived from that frame's surface height, top and bottom +reservation, including borders. This preserves the clip relationship +through interruptions; sharing progress alone would not guarantee it. +The editor retains responsibility for caret reveal, scroll and shaping. +The controller never edits the draft, selection or editor scroll offset. + +The natural header is measured inside a separate clip whose presented +height follows the same progress. The animated clip's height is never +fed back into the natural header measurement. Layout constraints and +compact/control sizes still come from `composer_layout.luau`. + +## Validation + +```sh +cargo test --locked --manifest-path apps/zgx/Cargo.toml --bin zeron-zgx composer +``` + +The targeted suite has 25 Rust tests, including bridges to eight layout +and four transition cases in Luau. It checks: + +- A single discrete width change in both pure geometry and the real editor; + continuous vertical geometry and clip/control bounds around that change. +- Retargeting growth and reversing shrink from the presented snapshot. +- Unchanged destinations retaining progress, including content above the cap. +- Context and Reduce motion snaps, including cancellation during a run. +- Unmount/remount, frozen disposed progress and no remaining frame requests. +- Real editor identity, focus and selection, typing during growth, actual + scrolling at the cap, and caret reveal after typing into that scrolled draft. + +For manual visual checks, the preview uses the real design-system composer +with local drafts and header data. Host engine/account updates cannot +replace those inputs: + +```sh +composer_preview_run=$(mktemp -d) +ZERON_ZGX_ENGINE_SPAWN=0 ZERON_ZGX_ENGINE_PORT=27963 \ +ZERON_ZGX_ENGINE_DATA="$composer_preview_run/engine" \ +cargo run --locked --manifest-path apps/zgx/Cargo.toml --bin zeron-zgx -- fixture=composer_transition_preview.luau --inspect +``` + +Use Short, 10 lines and 80 lines quickly to interrupt both directions. +Header toggles a real attachment. Wrap supplies a long line. Presets focus +the editor at the end; scroll up and type to check caret reveal. Repeat +with Reduce motion enabled and at different window widths/zoom levels. + +A local Linux X11/Xvfb run used 1200 × 800 content, 100% zoom, Zeron dark +and default frosted surfaces. A 12-second recording at 60 fps captured +720 frames, covering rapid interruptions, header changes, wrapping, +scroll/edit and Reduce motion. Transition samples and final captures were +visually inspected. An automated check found no entirely empty selected +text region across the 720 frames (minimum 95 grayscale pixels above 120). +This check excludes toolbar/footer/buttons but can miss local painting +errors; it is not a pixel-accurate rendering test. Local evidence is in +`.personal/composer-c4/` (ignored by Git). + +macOS Core Animation/Metal presentation was not exercised here. This work +does not establish a reproduction or confirmed fix of the original glitch. diff --git a/apps/zgx/src/composer_motion_tests.rs b/apps/zgx/src/composer_motion_tests.rs new file mode 100644 index 000000000..835d7f5b4 --- /dev/null +++ b/apps/zgx/src/composer_motion_tests.rs @@ -0,0 +1,174 @@ +use zgx_script::{ScriptHost, Value, testing::TestApp}; + +struct Controller { + app: TestApp, + frame: u32, +} + +impl Controller { + fn new() -> Self { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("ui/composer_transition_fixture.luau"); + let app = TestApp::new(|runtime| ScriptHost::new(runtime, path)); + app.ui.scene.borrow_mut().resize(800., 600.); + let mut c = Self { app, frame: 0 }; + c.frames(30); + c + } + + fn set(&self, key: &str, value: Value) { + self.app.host.set_state(key, value).unwrap(); + } + + fn height(&self, height: f64) { + self.set("fixture_transition_height", Value::Num(height)); + } + + fn frames(&mut self, n: usize) { + for _ in 0..n { + self.frame += 1; + self.set( + "fixture_transition_now", + Value::Num(f64::from(self.frame) * 1000. / 60.), + ); + self.app.frame(); + assert_eq!(self.app.error(), None); + } + } + + fn report(&self) -> Value { + assert!(self.app.host.run_command("fixture.transition.inspect")); + self.app + .host + .states() + .into_iter() + .find(|(n, _)| n == "fixture_transition_report") + .unwrap() + .1 + } +} + +fn num(v: &Value, field: &str) -> f64 { + v.field(field) + .and_then(Value::as_f64) + .unwrap_or_else(|| panic!("missing {field}: {v:?}")) +} + +#[test] +fn composer_transition_rules_cover_width_clip_headers_and_interruption() { + let ui = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("ui"); + let report = zgx_script::test_runner::run(&ui, Some("composer transition:"), |outcome| { + if let Some(failure) = &outcome.failure { + eprintln!("{}: {failure}", outcome.name); + } + }) + .unwrap(); + assert_eq!(report.outcomes.len(), 4); + assert_eq!(report.failed(), 0, "{report:?}"); +} + +#[test] +fn composer_transition_retargets_from_the_presented_frame_and_ignores_equal_destinations() { + let mut c = Controller::new(); + c.height(132.); + c.height(22.); // Reversed before the first display frame: already home. + assert_eq!(c.report().field("active"), Some(&Value::Bool(false))); + c.height(132.); + c.frames(3); + let before = c.report(); + assert_eq!(before.field("active"), Some(&Value::Bool(true))); + c.height(500.); + let replaced = c.report(); + assert_eq!( + replaced.field("origin"), + before.field("frame"), + "retarget must start at the presented snapshot" + ); + assert_eq!(num(&replaced, "progress"), 0.); + c.frames(3); + let growing = c.report(); + c.height(900.); // Same capped geometry, different natural content height. + let unchanged = c.report(); + assert_eq!(num(&unchanged, "progress"), num(&growing, "progress")); + assert_eq!(unchanged.field("origin"), growing.field("origin")); + c.frames(1); + assert!(num(&c.report(), "progress") > num(&growing, "progress")); + c.frames(30); + let settled = c.report(); + assert_eq!(settled.field("active"), Some(&Value::Bool(false))); + assert_eq!(settled.field("frame"), settled.field("destination")); + assert!( + !c.app.clock.wants_frame(), + "completed controller must stop requesting frames" + ); + c.height(22.); + c.frames(3); + let shrinking = c.report(); + assert_eq!(shrinking.field("active"), Some(&Value::Bool(true))); + c.height(180.); + assert_eq!( + c.report().field("origin"), + shrinking.field("frame"), + "reversing a shrink also starts at the presented snapshot" + ); + c.frames(30); + assert_eq!(c.report().field("frame"), c.report().field("destination")); + assert!(!c.app.clock.wants_frame()); +} + +#[test] +fn composer_transition_context_and_reduce_motion_snap_and_cancel_the_running_clip() { + let mut c = Controller::new(); + c.height(132.); + c.frames(3); + c.set("fixture_transition_snap", Value::Num(1.)); + let snapped = c.report(); + assert_eq!(snapped.field("frame"), snapped.field("destination")); + assert_eq!(snapped.field("active"), Some(&Value::Bool(false))); + c.height(500.); // A measurement settling during the context's snap window. + assert_eq!(c.report().field("frame"), c.report().field("destination")); + c.frames(30); + c.height(100.); + c.frames(3); + assert_eq!(c.report().field("active"), Some(&Value::Bool(true))); + c.set("motion_reduce", Value::Bool(true)); + let reduced = c.report(); + assert_eq!(reduced.field("frame"), reduced.field("destination")); + assert_eq!(reduced.field("active"), Some(&Value::Bool(false))); + c.frames(30); + assert_eq!( + c.report(), + reduced, + "cancelled callbacks cannot revive the old transition" + ); + assert!(!c.app.clock.wants_frame()); +} + +#[test] +fn composer_transition_unmount_freezes_progress_and_remount_owns_a_new_controller() { + let mut c = Controller::new(); + c.height(500.); + c.frames(3); + let before = c.report(); + c.set("fixture_transition_shown", Value::Bool(false)); + let disposed = c.report(); + assert_eq!(disposed.field("active"), Some(&Value::Bool(false))); + assert_eq!(num(&disposed, "progress"), num(&before, "progress")); + c.height(100.); + c.frames(30); + assert_eq!( + c.report(), + disposed, + "unmounted controller cannot keep updating" + ); + assert!(!c.app.clock.wants_frame()); + c.set("fixture_transition_shown", Value::Bool(true)); + let new = c.report(); + assert_eq!(num(&new, "progress"), 1.); + assert_eq!(new.field("frame"), new.field("destination")); + c.height(180.); + c.frames(3); + assert_eq!(c.report().field("active"), Some(&Value::Bool(true))); + c.frames(30); + assert_eq!(c.report().field("frame"), c.report().field("destination")); +} diff --git a/apps/zgx/src/main.rs b/apps/zgx/src/main.rs index fbf014057..1058f908b 100644 --- a/apps/zgx/src/main.rs +++ b/apps/zgx/src/main.rs @@ -80,6 +80,8 @@ mod composer_metrics_tests; #[cfg(test)] mod composer_layout_tests; #[cfg(test)] +mod composer_motion_tests; +#[cfg(test)] mod hit_sweep_tests; #[cfg(test)] mod hit_tests; diff --git a/apps/zgx/src/route_motion_tests.rs b/apps/zgx/src/route_motion_tests.rs index 8725883fa..25f3c000a 100644 --- a/apps/zgx/src/route_motion_tests.rs +++ b/apps/zgx/src/route_motion_tests.rs @@ -240,6 +240,74 @@ fn type_at_caret(app: &mut TestApp, text: &str) { app.settle(); } +#[test] +fn composer_transition_near_the_height_cap_preserves_scroll_editing_and_the_caret() { + for session in [false, true] { + let mut c = Composer::new(session); + c.act(|app| app.click(FIELD)); + let first = (0..10) + .map(|i| format!("line {i}")) + .collect::>() + .join("\n"); + c.act(|app| type_at_caret(app, &first)); + c.watch(3); + let more = (10..80).map(|i| format!("\nline {i}")).collect::(); + c.act(|app| type_at_caret(app, &more)); + c.trace.clear(); + c.watch(40); + let text = format!("{first}{more}"); + c.draft(&text, (text.len(), text.len())); + c.at_target(); + let cap = c.last().presented; + let field = c.shown.app.find(FIELD).unwrap(); + let editor = c.shown.app.ui.editor_within(field).unwrap(); + let caret_before = c + .shown + .app + .ui + .scene + .borrow() + .transformed_bounds(editor.caret); + c.act(|app| app.scroll(FIELD, -120.)); + c.watch(3); + let caret_scrolled = c + .shown + .app + .ui + .scene + .borrow() + .transformed_bounds(editor.caret); + assert!( + (caret_before.y - caret_scrolled.y).abs() > 20., + "the test must actually scroll the editor" + ); + c.act(|app| type_at_caret(app, " typed after scrolling")); + c.trace.clear(); + c.watch(20); + let text = format!("{text} typed after scrolling"); + c.draft(&text, (text.len(), text.len())); + for s in &c.trace { + assert_eq!( + s.presented.composer.height, cap.composer.height, + "typing above the cap must not resize the composer" + ); + assert_eq!(s.presented.field, cap.field); + } + let caret = c + .shown + .app + .ui + .scene + .borrow() + .transformed_bounds(editor.caret); + assert!( + caret.y >= cap.field.y - 1. + && caret.y + caret.height <= cap.field.y + cap.field.height + 1., + "typing must reveal the caret inside the viewport: {caret:?}, {cap:?}" + ); + } +} + #[test] fn composer_click_and_focus_preserve_geometry_and_the_selected_draft() { for session in [false, true] { @@ -300,6 +368,7 @@ fn composer_character_typing_crosses_the_wrap_boundary_and_folds_back() { c.act(|app| app.click(FIELD)); c.watch(1); let compact = c.last().target.composer.height; + let compact_width = c.last().presented.field.width; let mut text = String::new(); // Find the boundary with the actual font, without copying width rules. while c.last().target.composer.height <= compact + 1. { @@ -312,6 +381,24 @@ fn composer_character_typing_crosses_the_wrap_boundary_and_folds_back() { } c.watch(30); c.at_target(); + let expanded_width = c.last().target.field.width; + let mut width = compact_width; + let mut changes = 0; + for s in &c.trace { + let next = s.presented.field.width; + assert!( + (next - compact_width).abs() < 1. || (next - expanded_width).abs() < 1., + "mode change must not rewrap through intermediate widths: {s:?}" + ); + if (next - width).abs() >= 1. { + changes += 1; + } + width = next; + } + assert_eq!( + changes, 1, + "the real editor changes width once at expansion" + ); assert!(c.last().presented.composer.height > compact + 20.); while !text.is_empty() { c.act(|app| app.key(Key::Backspace)); diff --git a/apps/zgx/ui/composer_transition_fixture.luau b/apps/zgx/ui/composer_transition_fixture.luau new file mode 100644 index 000000000..adc76015a --- /dev/null +++ b/apps/zgx/ui/composer_transition_fixture.luau @@ -0,0 +1,39 @@ +-- Controller-only fixture: no engine, editor or unrelated animations. +local ds = require("@ds") +local Layout = require("@ds/composer_layout") +local Motion = require("@ds/composer_motion") +local clock = require("@ds/composer_clock") +ds.setup() +local now = ui.state("fixture_transition_now", 0) +clock.now = function() return now:peek() end +local height = ui.state("fixture_transition_height", 22) +local shown = ui.state("fixture_transition_shown", true) +local snap = ui.state("fixture_transition_snap", 0) +local report = ui.state("fixture_transition_report", {} :: any) +local target = ui.computed(function() + return Layout.calculate({ + width = 660, + sample = { unwrapped_width = 40, wrapped_height = height:get(), line_height = 22 }, + trailing_width = 64, control_corner = 25, + header_height = 0, header_gap = 12, + expanded = false, cargo = false, asking = false, dictating = false, + multiline = height:get() > 22, + }, false) +end) +local last: any = nil +ui.command { + id = "fixture.transition.inspect", + title = "Inspect the last controller, even after unmount", + run = function() + if last then + report:set({ frame = last.frame:peek(), origin = last.origin:peek(), + destination = last.destination:peek(), progress = last.progress:peek(), active = last.active:peek() }) + end + end, +} +local Controller = ui.component(function(): Node + last = Motion.use(target, snap) + local own = last + return ui.div { w = 660, h = function() return own.frame:get().height end } +end) +return ui.show(function() return shown:get() end, Controller {}) diff --git a/apps/zgx/ui/composer_transition_preview.luau b/apps/zgx/ui/composer_transition_preview.luau new file mode 100644 index 000000000..3768dec19 --- /dev/null +++ b/apps/zgx/ui/composer_transition_preview.luau @@ -0,0 +1,51 @@ +-- Real composer for visual transition checks. All input is fixture-local; +-- the development host's engine/account updates cannot replace this draft. +local ds = require("@ds") +local motion = require("@ds/motion") +ds.setup() +ui.theme.follow_system(false) +ui.theme.current:set("zeron-dark") +motion.reduce:set(false) + +local draft = ui.state("fixture_preview_draft", "short") +local focus = ui.state("fixture_preview_focus", 0) +local cargo = ui.state("fixture_preview_cargo", {} :: { any }) +local function lines(n: number): string + local out = {} + for i = 1, n do table.insert(out, `Line {i} for composer transition checks`) end + return table.concat(out, "\n") +end +local function set_draft(text: string) + ui.batch(function() + draft:set(text) + focus:set(focus:peek() + 1) + end) +end + +return ui.column { + w_full = true, h_full = true, p = "$space.4", gap = "$space.4", + bg = "$background", + ui.row { + gap = "$space.2", + ds.Button { "Short", on_click = function() set_draft("short") end }, + ds.Button { "10 lines", on_click = function() set_draft(lines(10)) end }, + ds.Button { "80 lines", on_click = function() set_draft(lines(80)) end }, + ds.Button { "Header", on_click = function() + cargo:set(if #cargo:peek() == 0 then { { id = "f", kind = "file", name = "notes.md" } } else {}) + focus:set(focus:peek() + 1) + end }, + ds.Button { "Reduce motion", on_click = function() motion.reduce:set(not motion.reduce:peek()) end }, + ds.Button { "Wrap", on_click = function() set_draft(string.rep("W", 60)) end }, + }, + ui.div { grow = 1 }, + ui.div { + w = 660, + ds.Composer { + value = draft, cargo = cargo, focus = focus, + model = ui.state("fixture_preview_model", "sonnet" :: string?), + effort = ui.state("fixture_preview_effort", "medium" :: string?), + harnesses = { { id = "claude", label = "Claude", mark = "claude" } }, + models = { { value = "sonnet", label = "Sonnet", harness = "claude" } }, + }, + }, +} diff --git a/apps/zgx/ui/ds/composer.luau b/apps/zgx/ui/ds/composer.luau index 72a8402bb..9f133f6ce 100644 --- a/apps/zgx/ui/ds/composer.luau +++ b/apps/zgx/ui/ds/composer.luau @@ -53,10 +53,9 @@ local ask = require("@self/composer_ask") local mentions = require("@self/composer_mentions") local locale = require("@self/locale") local context_usage = require("@self/context_usage") -local motion = require("@self/motion") -local clock = require("@ds/composer_clock") local TextMetrics = require("@self/composer_metrics") local Layout = require("@self/composer_layout") +local ComposerMotion = require("@self/composer_motion") local EditorMenu = require("@self/editor_menu") -- What the model's context holds, as its agent reports it: the tokens in @@ -1095,48 +1094,13 @@ return ui.component(function(props: Props): Node target_geometry:set(next) end) end) - local EASE: TweenOptions = { duration = 220, ease = "out(3)" } - -- Each of the pill's measures as it shows: eased (EASE) as it grows or - -- shrinks in place; at once for a moment after `snap` moves (what - -- settles its size, measured, lands over a frame or two), and with - -- motion reduced. - local SNAP_MS = 400 - local snap_seen = if props.snap then props.snap:peek() else nil - local snap_until = 0 - local function at_once(): boolean - if props.snap then - local n = props.snap:get() - if n ~= snap_seen then - snap_seen = n - snap_until = clock.now() + SNAP_MS - end + local transition = ComposerMotion.use(target_geometry, props.snap) + local frame_geometry = transition.frame + -- Every binding reads the same frame snapshot; none starts an animation. + local function presented(select: (Layout.Geometry) -> number): () -> number + return function(): number + return select(frame_geometry:get()) end - return motion.reduced_now() or clock.now() < snap_until - end - local function eased(src: ReadOnlyState): State - local value = ui.use_state(src:peek()) - local run: any = nil - ui.effect(function() - local v = src:get() - local now = at_once() - if run then - run:cancel() - run = nil - end - if now or math.abs(value:peek() - v) < 0.01 then - value:set(v) - return - end - run = anime.animate(value, { value = v, duration = EASE.duration, ease = EASE.ease }) - end) - return value - end - -- C3 centralizes destinations only. Each scalar still has the existing - -- independent animation; coordinating those is the next change (C4). - local function presented(select: (Layout.Geometry) -> number): State - return eased(ui.computed(function(): number - return select(target_geometry:get()) - end)) end -- +, as a secondary button: a fill of its own, so it reads as a control @@ -1707,7 +1671,14 @@ return ui.component(function(props: Props): Node }, metrics.nodes[1], metrics.nodes[2], - head, + -- Measure the natural header inside a separate presented clip. The + -- clip's animated height must never feed the header measurement. + ui.div { + absolute = true, left = 0, right = 0, top = 0, + h = presented(function(g) return g.header_height end), + overflow = "hidden", + head, + }, -- The draft. ui.div { absolute = true, @@ -1876,7 +1847,7 @@ return ui.component(function(props: Props): Node right = presented(function(g) return g.controls.trailing.right end), -- (Its row is 32: the control centred in it.) bottom = presented(function(g) return g.controls.trailing.bottom end), - h = function() return target_geometry:get().controls.trailing.height end, + h = presented(function(g) return g.controls.trailing.height end), items = "center", gap = "$space.2", ui.show(asking, ask_controls()), @@ -2106,7 +2077,7 @@ return ui.component(function(props: Props): Node return if on_tray() then `${TRAY_GAP}` else 0 end :: any, px = function(): any - return if on_tray() then ui.raw(target_geometry:get().tray_inset, "the pill's controls' inset") else "$zeron.space.snug" + return if on_tray() then ui.raw(frame_geometry:get().tray_inset, "the pill's controls' inset") else "$zeron.space.snug" end :: any, items = "center", gap = "$space.1", diff --git a/apps/zgx/ui/ds/composer_layout.luau b/apps/zgx/ui/ds/composer_layout.luau index 80a0ffeba..d171751f0 100644 --- a/apps/zgx/ui/ds/composer_layout.luau +++ b/apps/zgx/ui/ds/composer_layout.luau @@ -27,6 +27,7 @@ export type Input = { multiline: boolean, } export type Geometry = { + width: number, mode: "compact" | "expanded", expanded: boolean, next_text_expanded: boolean, @@ -34,9 +35,13 @@ export type Geometry = { height: number, header_height: number, tray_inset: number, + -- Constraints for deriving a safe viewport from a presented frame. + line_height: number, + compact_inset: number, + compact_text: { left: number, right: number }, text: { left: number, right: number, width: number, top: number, height: number }, controls: { - leading: { left: number, bottom: number }, + leading: { left: number, bottom: number, height: number }, trailing: { right: number, bottom: number, height: number }, voice: { left: number, right: number, bottom: number }, }, @@ -91,6 +96,7 @@ function M.calculate(p: Input, previous_text_expanded: boolean): Geometry local right = if expanded then TEXT_OPEN_SIDE else line_right local inset = if expanded then chrome.inset else INSET_LINE return { + width = math.max(0, p.width), mode = if expanded then "expanded" else "compact", expanded = expanded, next_text_expanded = next_text_expanded, @@ -100,6 +106,9 @@ function M.calculate(p: Input, previous_text_expanded: boolean): Geometry height = height, header_height = head, tray_inset = chrome.tray_inset, + line_height = if p.sample then p.sample.line_height else LINE, + compact_inset = INSET_LINE, + compact_text = { left = line_left, right = line_right }, text = { left = left, right = right, @@ -108,7 +117,7 @@ function M.calculate(p: Input, previous_text_expanded: boolean): Geometry height = if expanded then box - TEXT_TOP - TEXT_BOTTOM else LINE, }, controls = { - leading = { left = inset, bottom = inset }, + leading = { left = inset, bottom = inset, height = CONTROL }, trailing = { right = inset, bottom = inset - 2, height = 32 }, voice = { left = inset + CONTROL + TEXT_SIDE, diff --git a/apps/zgx/ui/ds/composer_motion.luau b/apps/zgx/ui/ds/composer_motion.luau new file mode 100644 index 000000000..c0c0f20ab --- /dev/null +++ b/apps/zgx/ui/ds/composer_motion.luau @@ -0,0 +1,168 @@ +-- One progress and one complete geometry per frame. The editor stays +-- compact until a whole line fits above the controls, then takes its final +-- expanded width in one step. It never wraps through interpolated insets. +local Layout = require("@ds/composer_layout") +local M = {} + +local function mix(a: number, b: number, p: number): number + return a + (b - a) * p +end + +function M.frame(from: Layout.Geometry, to: Layout.Geometry, progress: number): Layout.Geometry + local p = math.clamp(progress, 0, 1) + if p == 1 then + return to + end + local g = table.clone(to) + g.height = mix(from.height, to.height, p) + g.header_height = mix(from.header_height, to.header_height, p) + g.controls = { + leading = { + left = mix(from.controls.leading.left, to.controls.leading.left, p), + bottom = mix(from.controls.leading.bottom, to.controls.leading.bottom, p), + height = to.controls.leading.height, + }, + trailing = { + right = mix(from.controls.trailing.right, to.controls.trailing.right, p), + bottom = mix(from.controls.trailing.bottom, to.controls.trailing.bottom, p), + height = to.controls.trailing.height, + }, + voice = { + left = mix(from.controls.voice.left, to.controls.voice.left, p), + right = mix(from.controls.voice.right, to.controls.voice.right, p), + bottom = mix(from.controls.voice.bottom, to.controls.voice.bottom, p), + }, + } + -- Animate the vertical envelope continuously, deriving the viewport + -- from the current container and reserved space. Switching width must + -- not also jump the text vertically or leave it over the buttons. + local top = mix(from.text.top, to.text.top, p) + local from_bottom = from.height - Layout.BORDER * 2 - from.text.top - from.text.height + local to_bottom = to.height - Layout.BORDER * 2 - to.text.top - to.text.height + local bottom = mix(from_bottom, to_bottom, p) + local available = math.max(0, g.height - Layout.BORDER * 2 - top - bottom) + local controls = math.max(g.controls.leading.bottom + g.controls.leading.height, + g.controls.trailing.bottom + g.controls.trailing.height) + local expand = to.expanded and bottom >= controls - 0.001 + and available >= math.min(to.line_height, to.text.height) - 0.001 + local left = if expand then to.text.left else to.compact_text.left + local right = if expand then to.text.right else to.compact_text.right + if not expand then + -- Keep compact text and controls beside each other, including themes + -- with deeply inset expanded controls. Their vertical anchors move. + g.controls.leading.left = to.compact_inset + g.controls.trailing.right = to.compact_inset + end + g.text = { + left = left, right = right, + width = math.max(0, to.width - Layout.BORDER * 2 - left - right), + top = top, height = available, + } + g.expanded = expand + g.mode = if expand then "expanded" else "compact" + return g +end + +-- Changes in natural content height at the viewport cap, or in hysteresis +-- metadata, don't change the destination and must not restart a run. +function M.same_destination(a: Layout.Geometry, b: Layout.Geometry): boolean + if a.width ~= b.width or a.height ~= b.height or a.mode ~= b.mode or a.expanded ~= b.expanded + or a.header_height ~= b.header_height then + return false + end + for name, value in a.text do + if (b.text :: any)[name] ~= value then + return false + end + end + for key, anchors in a.controls do + for name, value in anchors do + if (b.controls :: any)[key][name] ~= value then + return false + end + end + end + return true +end + +function M.use(target: ReadOnlyState, snap: State?) + local motion = require("@self/motion") + local clock = require("@self/composer_clock") + local initial = target:peek() + local origin = ui.use_state(initial) + local destination = ui.use_state(initial) + local progress = ui.use_state(1) + local active = ui.use_state(false) + local frame = ui.computed(function(): Layout.Geometry + return M.frame(origin:get(), destination:get(), progress:get()) + end) + local run: any = nil + local alive, generation = true, 0 + local snap_seen = if snap then snap:peek() else nil + local snap_until = 0 + local function cancel() + generation += 1 + if run then + run:cancel() + run = nil + end + active:set(false) + end + local function settle(next: Layout.Geometry) + cancel() + ui.batch(function() + origin:set(next) + destination:set(next) + progress:set(1) + end) + end + ui.effect(function() + local next = target:get() + local reduced = motion.reduced() + if snap then + local n = snap:get() + if n ~= snap_seen then + snap_seen = n + -- Keep C3's context-settle window; replacing deadlines is C6. + snap_until = clock.now() + 400 + end + end + if reduced or clock.now() < snap_until then + settle(next) + return + end + if M.same_destination(destination:peek(), next) then + destination:set(next) + return + end + local shown = frame:peek() + if M.same_destination(shown, next) then + settle(next) + return + end + cancel() + local own = generation + ui.batch(function() + origin:set(shown) + destination:set(next) + progress:set(0) + active:set(true) + end) + run = anime.animate(progress, { + value = 1, duration = 220, ease = "out(3)", + onComplete = function() + if alive and generation == own then + run = nil + active:set(false) + end + end, + }) + end) + ui.on_cleanup(function() + alive = false + cancel() + end) + return { frame = frame, origin = origin, destination = destination, progress = progress, active = active } +end + +return M diff --git a/apps/zgx/ui/tests/composer_layout.test.luau b/apps/zgx/ui/tests/composer_layout.test.luau index d61742e56..7ac156734 100644 --- a/apps/zgx/ui/tests/composer_layout.test.luau +++ b/apps/zgx/ui/tests/composer_layout.test.luau @@ -26,7 +26,7 @@ test("composer geometry: compact destination and measured controls", function() expect(g.height).to_be(49) expect(g.natural_height).to_be(49) expect(g.text).to_equal({ left = 45.5, right = 81.5, top = 12.5, width = 531, height = 22 }) - expect(g.controls.leading).to_equal({ left = 9.5, bottom = 9.5 }) + expect(g.controls.leading).to_equal({ left = 9.5, bottom = 9.5, height = 28 }) expect(g.controls.trailing).to_equal({ right = 9.5, bottom = 7.5, height = 32 }) local more_controls = Layout.calculate(input({ trailing_width = 100 }), false) expect(more_controls.text.width).to_be(495) diff --git a/apps/zgx/ui/tests/composer_motion.test.luau b/apps/zgx/ui/tests/composer_motion.test.luau new file mode 100644 index 000000000..01ce51899 --- /dev/null +++ b/apps/zgx/ui/tests/composer_motion.test.luau @@ -0,0 +1,74 @@ +local Layout = require("@ds/composer_layout") +local Motion = require("@ds/composer_motion") + +local function geometry(height: number, header: number?, corner: number?) + return Layout.calculate({ + width = 660, sample = { unwrapped_width = 40, wrapped_height = height, line_height = 22 }, + trailing_width = 64, control_corner = corner or 25, + header_height = header or 0, header_gap = 12, + expanded = false, cargo = (header or 0) > 0, asking = false, dictating = false, + multiline = height > 22, + }, false) +end + +test("composer transition: width changes once after a full expanded line fits", function() + local a, b = geometry(22), geometry(220) + local changes, last = 0, a.text.width + local previous = Motion.frame(a, b, 0) + for i = 0, 100 do + local g = Motion.frame(a, b, i / 100) + expect(g.text.width == a.text.width or g.text.width == b.text.width).to_be(true) + if g.text.width ~= last then + changes += 1 + -- A discrete width change must not also jump the vertical field. + expect(math.abs(g.text.top - previous.text.top) < 1).to_be(true) + expect(math.abs(g.text.height - previous.text.height) < 5).to_be(true) + end + last = g.text.width + previous = g + expect(g.text.height >= 22).to_be(true) + expect(g.text.top >= 0).to_be(true) + expect(g.text.top + g.text.height <= g.height - Layout.BORDER * 2 + 0.001).to_be(true) + if g.expanded then + expect(g.text.top + g.text.height < g.height - g.controls.trailing.bottom - g.controls.trailing.height).to_be(true) + end + end + expect(changes).to_be(1) + expect(Motion.frame(a, b, 1)).to_equal(b) +end) + +test("composer transition: headers and rounder controls remain inside a valid frame", function() + local a, b = geometry(22), geometry(500, 90, 45) + for _, pair in { { a, b }, { b, a } } do + for i = 0, 100 do + local g = Motion.frame(pair[1], pair[2], i / 100) + expect(g.header_height <= g.text.top + 0.001).to_be(true) + expect(g.text.height >= 22).to_be(true) + expect(g.text.top + g.text.height <= g.height - 2 + 0.001).to_be(true) + if not g.expanded then + expect(g.controls.leading.left + 28 < g.text.left).to_be(true) + end + end + end +end) + +test("composer transition: interrupted height and anchors start at the presented geometry", function() + local shown = Motion.frame(geometry(22), geometry(500, 90), 0.3) + local g = Motion.frame(shown, geometry(100, 30), 0) + expect(g.height).to_be(shown.height) + expect(g.header_height).to_be(shown.header_height) + expect(g.controls.leading.bottom).to_be(shown.controls.leading.bottom) + expect(g.controls.trailing.bottom).to_be(shown.controls.trailing.bottom) +end) + +test("composer transition: changes above the cap do not change the visual destination", function() + local a, b = geometry(500), geometry(900) + expect(a.natural_height == b.natural_height).to_be(false) + expect(Motion.same_destination(a, b)).to_be(true) + local constraints = table.clone(a) + constraints.line_height = 24 + constraints.compact_text = { left = 45.5, right = 100 } + expect(Motion.same_destination(a, constraints)).to_be(true) + expect(Motion.same_destination(a, geometry(100))).to_be(false) + expect(Motion.same_destination(a, geometry(500, 30))).to_be(false) +end) From 6bb272fdcb5daed08c6d7947e32e87230db13648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gurruchaga?= Date: Sun, 8 Nov 2026 19:15:35 -0300 Subject: [PATCH 10/81] refactor(zgx): separate composer placement from sizing --- apps/zgx/docs/composer-layout.md | 12 ++++ apps/zgx/src/route_motion_tests.rs | 54 ++++++++++++++ apps/zgx/ui/ds/composer.luau | 26 ++++++- apps/zgx/ui/ds/init.luau | 1 + apps/zgx/ui/screens/chat/composer.luau | 72 ++++++------------- apps/zgx/ui/screens/chat/composer_place.luau | 13 ++++ apps/zgx/ui/screens/chat/lane_tab.luau | 7 +- apps/zgx/ui/templates/zeron-v2/.base.json | 4 +- .../zeron-v2/screens/chat/composer.luau | 72 ++++++------------- .../zeron-v2/screens/chat/lane_tab.luau | 7 +- 10 files changed, 158 insertions(+), 110 deletions(-) create mode 100644 apps/zgx/ui/screens/chat/composer_place.luau diff --git a/apps/zgx/docs/composer-layout.md b/apps/zgx/docs/composer-layout.md index bfe1765a7..17996f9a2 100644 --- a/apps/zgx/docs/composer-layout.md +++ b/apps/zgx/docs/composer-layout.md @@ -52,6 +52,18 @@ read a single presented-frame snapshot rather than animating each scalar. ## Target, presented and published sizes +C5 adds `Composer.on_sizes`: `rest` is calculated with an empty draft and +no cargo/ask/dictation; `target` includes the destination pill, measured +deck and footer; `presented` is the laid-out root. `active` and `ready` +describe motion and metric validity. The footer/deck are measured separately, +so animated pill height is never used to infer the empty height. Main and +side composers publish the same contract to their transcript reservation. + +`screens/chat/composer_place.luau` handles page placement only. A new chat +uses the rest height to retain its top and the presented height to avoid +overflow. The page still owns dock/center placement, send glide and the +model-picker hold. Clearing a draft no longer waits 500 ms to infer rest. + `target_geometry` describes the destination, not an intermediate frame. `composer_motion.luau` (C4) owns origin, destination and a single progress animation. The `presented` helper only selects values from that controller's diff --git a/apps/zgx/src/route_motion_tests.rs b/apps/zgx/src/route_motion_tests.rs index 25f3c000a..4c403b63c 100644 --- a/apps/zgx/src/route_motion_tests.rs +++ b/apps/zgx/src/route_motion_tests.rs @@ -240,6 +240,60 @@ fn type_at_caret(app: &mut TestApp, text: &str) { app.settle(); } +#[test] +fn composer_rest_size_is_independent_of_animated_draft_size_and_picker_placement() { + let mut c = Composer::new(false); + let sizes = |app: &TestApp| { + app.host + .states() + .into_iter() + .find(|(n, _)| n.ends_with("::chat_composer_sizes")) + .unwrap() + .1 + }; + let rest = sizes(&c.shown.app).field("rest").unwrap().as_f64().unwrap(); + c.act(|app| app.fill(FIELD, "one\ntwo\nthree\nfour\nfive\nsix")); + c.watch(3); + let moving = sizes(&c.shown.app); + assert_eq!(moving.field("rest").unwrap().as_f64(), Some(rest)); + assert!(moving.field("target").unwrap().as_f64().unwrap() > rest); + assert!( + moving.field("presented").unwrap().as_f64().unwrap() + < moving.field("target").unwrap().as_f64().unwrap() + ); + c.watch(40); + c.act(|app| app.fill(FIELD, "")); + c.watch(1); + assert_eq!( + sizes(&c.shown.app).field("rest").unwrap().as_f64(), + Some(rest), + "clearing needs no settle timer" + ); + c.watch(40); + c.act(|app| app.click("#chat-composer >> role:button \"Model\"")); + c.watch(2); + let button = c + .shown + .app + .find("#chat-composer >> role:button \"Model\"") + .unwrap(); + let y = c.shown.app.ui.scene.borrow().transformed_bounds(button).y; + c.act(|app| { + app.host + .set_state( + "chat.composer.text", + Value::from("one\ntwo\nthree\nfour\nfive\nsix"), + ) + .unwrap() + }); + c.watch(40); + let held = c.shown.app.ui.scene.borrow().transformed_bounds(button).y; + assert!( + (held - y).abs() < 1., + "open picker retains its anchor: {y} -> {held}" + ); +} + #[test] fn composer_transition_near_the_height_cap_preserves_scroll_editing_and_the_caret() { for session in [false, true] { diff --git a/apps/zgx/ui/ds/composer.luau b/apps/zgx/ui/ds/composer.luau index 9f133f6ce..78feb6653 100644 --- a/apps/zgx/ui/ds/composer.luau +++ b/apps/zgx/ui/ds/composer.luau @@ -89,7 +89,9 @@ export type Notice = { export type Answer = { values: { string }, text: string } -- Where an existing chat works (`branch_fixed`). export type Fixed = { device: string?, name: string?, worktree: boolean, project: string?, hue: string?, src: string?, emoji: string? } +export type Sizes = { rest: number, target: number, presented: number, active: boolean, ready: boolean } export type Props = { + on_sizes: ((sizes: Sizes) -> ())?, value: State, placeholder: (string | Message)?, -- What sends (the shipping app's setting): "enter", ↩ sends and ⇧↩ @@ -2189,6 +2191,7 @@ return ui.component(function(props: Props): Node -- The pill (the mentions' popover anchored on it: its panel opens at -- the caret inside), its trays behind, the footer under. + local footer_size, deck_size, outer_size = ui.ref(), ui.ref(), ui.ref() local body = ui.column { w_full = true, relative = true, @@ -2267,10 +2270,10 @@ return ui.component(function(props: Props): Node return if material.see_through() then "$zeron.composer.rest" else "$zeron.composer.shadow" end, }, - trays, + ui.column { ref = deck_size, w_full = true, shrink = 0, trays }, ui.popover(m.popover(surface)), }, - footer, + ui.column { ref = footer_size, w_full = true, shrink = 0, footer }, -- Folded, what waits on the agent: a pill floating just over the -- composer's middle, out of the layout (the conversation shows under -- it, as under "Scroll to end", which it moves aside for). @@ -2288,7 +2291,26 @@ return ui.component(function(props: Props): Node else nil, } + if props.on_sizes then + local on_sizes = props.on_sizes + ui.effect(function() + local input = layout_input() + input.sample = nil + input.cargo, input.asking, input.dictating, input.multiline = false, false, false, false + local rest = Layout.calculate(input, false).height + local footer_h, deck_h, actual = footer_size.height, deck_size.height, outer_size.height + local target = target_geometry:get() + local active = transition.active:get() + local ready = metrics.result:get().status == "ready" + if footer_h ~= nil and deck_h ~= nil and actual and actual > 0 then + on_sizes({ rest = rest + footer_h + deck_h, target = target.height + footer_h + deck_h, + presented = actual, active = active, ready = ready }) + end + end) + end + return ui.column { + ref = outer_size, w_full = true, gap = 0, body, diff --git a/apps/zgx/ui/ds/init.luau b/apps/zgx/ui/ds/init.luau index 55b140a14..0722a344a 100644 --- a/apps/zgx/ui/ds/init.luau +++ b/apps/zgx/ui/ds/init.luau @@ -26,6 +26,7 @@ export type SidebarProject = sidebar.Project export type StatusKind = status.Kind export type Harness = model_picker.Harness export type Model = model_picker.Model +export type ComposerSizes = composer.Sizes export type ComposerDevice = composer.Device export type ComposerUsage = composer.Usage export type ComposerFixed = composer.Fixed diff --git a/apps/zgx/ui/screens/chat/composer.luau b/apps/zgx/ui/screens/chat/composer.luau index 850ac2c7e..a4c78f151 100644 --- a/apps/zgx/ui/screens/chat/composer.luau +++ b/apps/zgx/ui/screens/chat/composer.luau @@ -17,6 +17,7 @@ local ds = require("@ds") local motion = require("@ds/motion") +local Placement = require("@app/screens/chat/composer_place") local clock = require("@ds/composer_clock") local T = require("@ds/transcript") local prefs = require("@app/screens/settings/prefs") @@ -811,8 +812,15 @@ local function Composer(): Node } end + local composer_sizes = ui.state("chat_composer_sizes", nil :: ds.ComposerSizes?) local function composer(): Node return ds.Composer { + on_sizes = function(sizes: ds.ComposerSizes) + composer_sizes:set(sizes) + if math.abs(shared.dock_h:peek() - sizes.presented) > 0.01 then + shared.dock_h:set(sizes.presented) + end + end, -- Its measures, fold and open tray kept across a reload, so it -- is laid out as it was from the first frame. keep = "chat", @@ -993,40 +1001,14 @@ local function Composer(): Node ui.on_cleanup(function() shared.composer_shown:set(false) end) - local dock = ui.ref() - -- How tall a new chat's composer is with nothing in it (what it's placed - -- by, so what's written in it doesn't move its top). - local rest_tall = ui.state("chat_composer_rest_tall", nil :: number?) - -- (A draft just deleted, it's empty while still easing back, and that - -- height would move its top: taken once that's over.) - local REST_SETTLE_MS = 500 - local had_draft = false - local rest_settles_at = 0 - ui.effect(function() - local tall = dock.height or 0 - if shared.text:get() ~= "" or #shared.cargo:get() > 0 or next(shared.quotes:get()) ~= nil then - had_draft = true - return - end - if had_draft then - had_draft = false - rest_settles_at = clock.now() + REST_SETTLE_MS - end - if tall <= 0 or not new_chat() then - return - end - local function take() - if math.abs((rest_tall:peek() or 0) - tall) >= 0.5 then - rest_tall:set(tall) - end - end - local left = rest_settles_at - clock.now() - if left > 0 and rest_tall:peek() ~= nil then - ui.after(left, take) - else - take() - end + local rest_tall = ui.computed(function(): number? + local sizes = composer_sizes:get() + return if sizes then sizes.rest else nil end) + local function presented_height(): number + local sizes = composer_sizes:get() + return if sizes then sizes.presented else shared.dock_h:get() + end -- Where it stands: mid-page on a new chat (centred under the title bar), -- else docked at the foot (0), as laid out now. local function target_y(): number @@ -1039,7 +1021,7 @@ local function Composer(): Node if h <= 0 then h = page_h:get() end - local tall = dock.height or 0 + local tall = presented_height() if tall <= 0 then tall = shared.dock_h:get() end @@ -1047,9 +1029,7 @@ local function Composer(): Node -- and grows down from there. Only a draft that would run off the -- page's foot moves it, up by as much.) local rest = rest_tall:get() or tall - local hero = TITLE_H + (h - TITLE_H - 96) / 2 - local docked = h - FOOT - rest / 2 - return math.min(hero - docked, rest - tall) + return Placement.dock_offset(h, rest, tall, FOOT) end -- Moving between a new chat and a session: its own send from a new chat -- slides it down with the prompt (`slide_next`); any other move (another @@ -1112,7 +1092,7 @@ local function Composer(): Node gliding = nil end place:set(y) - local tall = dock.height or 0 + local tall = presented_height() if model_open:get() then held_tall = held_tall or tall hold:set(held_tall - tall) @@ -1200,21 +1180,15 @@ local function Composer(): Node if not new_chat() then return place:get() end - local tall = dock.height or 0 + local tall = presented_height() if tall <= 0 then tall = shared.dock_h:get() end local rest = rest_tall:get() or tall - local at = TITLE_H / 2 - 48 - rest / 2 - -- (Only a draft that would run off the page's foot moves it.) - local h = page.height or 0 - if h > 0 then - at = math.min(at, h / 2 - FOOT - tall) - end + local at = Placement.new_offset(page.height or 0, rest, tall, FOOT) return at + hold:get() end, ui.div { - ref = dock, id = "chat-composer", hook = "chat.composer", -- (Over the page, not in it: the session it sends to, as the @@ -1228,11 +1202,7 @@ local function Composer(): Node end), DOCK ), - on_resize = function(size: { w: number, h: number }) - if size.h >= 1 and math.abs(shared.dock_h:peek() - size.h) > 0.01 then - shared.dock_h:set(size.h) - end - end, + -- Files carried over the composer itself: the chat's drop, -- as over the page under it. on_file_hover = composer_zone.on_file_hover, diff --git a/apps/zgx/ui/screens/chat/composer_place.luau b/apps/zgx/ui/screens/chat/composer_place.luau new file mode 100644 index 000000000..ce360e4d2 --- /dev/null +++ b/apps/zgx/ui/screens/chat/composer_place.luau @@ -0,0 +1,13 @@ +-- Placement only: dimensions come from the composer's size contract. +local M = {} + +function M.new_offset(page: number, rest: number, presented: number, foot: number): number + local centered = 38 / 2 - 48 - rest / 2 + return if page > 0 then math.min(centered, page / 2 - foot - presented) else centered +end + +function M.dock_offset(page: number, rest: number, presented: number, foot: number): number + return page / 2 + M.new_offset(page, rest, presented, foot) + presented - (page - foot) +end + +return M diff --git a/apps/zgx/ui/screens/chat/lane_tab.luau b/apps/zgx/ui/screens/chat/lane_tab.luau index 357698a32..29f26d65c 100644 --- a/apps/zgx/ui/screens/chat/lane_tab.luau +++ b/apps/zgx/ui/screens/chat/lane_tab.luau @@ -198,7 +198,7 @@ end -- what's picked (a changed agent, model or access becomes the chat's, as the -- chat page's does). A message that didn't go says so on the composer's own -- notice line, above it, never over it. -local function composer(chat: string, view: any, name: string, focus: State, open_model: State, split: boolean?, focused: (() -> boolean)?): Node +local function composer(chat: string, view: any, name: string, focus: State, open_model: State, split: boolean?, focused: (() -> boolean)?, published_height: State?): Node -- (Its draft kept by the chat, as its picks: a switch of tabs or a -- reload finds it as it was; a passage quoted from its transcript lands -- in it.) @@ -607,6 +607,9 @@ local function composer(chat: string, view: any, name: string, focus: State