From c107bc23ee6f6f85d5179b1483d3d895e6dac7d9 Mon Sep 17 00:00:00 2001 From: Ankur Goyal Date: Thu, 5 Feb 2026 23:38:50 -0800 Subject: [PATCH 1/2] wip --- Cargo.lock | 1 + crates/lingua/Cargo.toml | 1 + .../tests/fuzz_roundtrip.proptest-regressions | 11 + crates/lingua/tests/fuzz_roundtrip.rs | 681 ++++++++++++++++++ crates/lingua/tests/roundtrip_cases.json | 159 ++++ 5 files changed, 853 insertions(+) create mode 100644 crates/lingua/tests/fuzz_roundtrip.proptest-regressions create mode 100644 crates/lingua/tests/fuzz_roundtrip.rs create mode 100644 crates/lingua/tests/roundtrip_cases.json diff --git a/Cargo.lock b/Cargo.lock index b3d1f4d..0b5192e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2080,6 +2080,7 @@ dependencies = [ "paste", "pbjson", "pbjson-types", + "proptest", "prost", "prost-types", "pyo3", diff --git a/crates/lingua/Cargo.toml b/crates/lingua/Cargo.toml index d9fa869..1567d59 100644 --- a/crates/lingua/Cargo.toml +++ b/crates/lingua/Cargo.toml @@ -58,6 +58,7 @@ tokio-test.workspace = true paste.workspace = true log.workspace = true env_logger.workspace = true +proptest.workspace = true [lib] name = "lingua" diff --git a/crates/lingua/tests/fuzz_roundtrip.proptest-regressions b/crates/lingua/tests/fuzz_roundtrip.proptest-regressions new file mode 100644 index 0000000..0941aec --- /dev/null +++ b/crates/lingua/tests/fuzz_roundtrip.proptest-regressions @@ -0,0 +1,11 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 571bebd27a681a4afddc88a50ab68906c2411c4653316c807667968e1da77dbf # shrinks to content = Array([Text(TextContentPart { text: "a", provider_options: None })]) +cc 9c78a0e43216173390bc5704b23d8c3f03e950c41b292798b57e71ffa7b3e460 # shrinks to req = UniversalRequest { model: None, messages: [System { content: Array([Text(TextContentPart { text: "0", provider_options: None })]) }], params: UniversalParams { temperature: None, top_p: None, top_k: None, seed: None, presence_penalty: None, frequency_penalty: None, max_tokens: None, stop: None, logprobs: None, top_logprobs: None, tools: None, tool_choice: None, parallel_tool_calls: None, response_format: None, reasoning: None, metadata: None, store: None, service_tier: None, stream: None, extras: {} } } +cc f4f66bfaf60b020ad8dd448344f0127d0ffbf59ce9d042ee4755eb43d1de9131 # shrinks to req = UniversalRequest { model: None, messages: [Tool { content: [ToolResult(ToolResultContentPart { tool_call_id: "call_0aA00a00", tool_name: "___", output: Null, provider_options: None }), ToolResult(ToolResultContentPart { tool_call_id: "call_A0aaaaAA", tool_name: "aaa", output: Null, provider_options: None })] }], params: UniversalParams { temperature: None, top_p: None, top_k: None, seed: None, presence_penalty: None, frequency_penalty: None, max_tokens: None, stop: None, logprobs: None, top_logprobs: None, tools: None, tool_choice: None, parallel_tool_calls: None, response_format: None, reasoning: None, metadata: None, store: None, service_tier: None, stream: None, extras: {} } } +cc 3b06f6ee7178d80cdcf18873d071b554ef516e474543d66deee9c2bc3e2dcaec # shrinks to req = UniversalRequest { model: None, messages: [Tool { content: [ToolResult(ToolResultContentPart { tool_call_id: "call_0aaAAAA0", tool_name: "aa_", output: Null, provider_options: None }), ToolResult(ToolResultContentPart { tool_call_id: "call_0aaAAAaa", tool_name: "___", output: Null, provider_options: None })] }], params: UniversalParams { temperature: None, top_p: None, top_k: None, seed: None, presence_penalty: None, frequency_penalty: None, max_tokens: None, stop: None, logprobs: None, top_logprobs: None, tools: None, tool_choice: None, parallel_tool_calls: None, response_format: None, reasoning: None, metadata: None, store: None, service_tier: None, stream: None, extras: {} } } +cc 992d77a84f2c6777ee72c0183797d3015725b3d04feee90c992a94d4f5478aea # shrinks to req = UniversalRequest { model: None, messages: [Tool { content: [ToolResult(ToolResultContentPart { tool_call_id: "call_a00Aaa0a", tool_name: "a_a", output: Null, provider_options: None }), ToolResult(ToolResultContentPart { tool_call_id: "call_A0Aa00a0", tool_name: "_a_", output: Null, provider_options: None })] }], params: UniversalParams { temperature: None, top_p: None, top_k: None, seed: None, presence_penalty: None, frequency_penalty: None, max_tokens: None, stop: None, logprobs: None, top_logprobs: None, tools: None, tool_choice: None, parallel_tool_calls: None, response_format: None, reasoning: None, metadata: None, store: None, service_tier: None, stream: None, extras: {} } } diff --git a/crates/lingua/tests/fuzz_roundtrip.rs b/crates/lingua/tests/fuzz_roundtrip.rs new file mode 100644 index 0000000..ba57dd9 --- /dev/null +++ b/crates/lingua/tests/fuzz_roundtrip.rs @@ -0,0 +1,681 @@ +//! Property-based roundtrip tests for Lingua cross-provider conversions. +//! +//! Test cases come from two sources: +//! 1. **Saved cases** (`roundtrip_cases.json`): Hand-written provider JSON payloads in any +//! format (OpenAI, Anthropic, Google, etc.). When proptest finds a novel failure, copy +//! the provider JSON into this file to make it a permanent regression test. +//! 2. **Proptest strategies**: Random `UniversalRequest` generation for coverage. +//! +//! The roundtrip harness is format-agnostic: it accepts either a `UniversalRequest` directly +//! or a `(ProviderFormat, serde_json::Value)` pair and runs the same assertions. + +use lingua::processing::adapter_for_format; +use lingua::serde_json::{self, json, Value}; +use lingua::universal::message::*; +use lingua::{ProviderFormat, UniversalParams, UniversalRequest}; +use std::path::PathBuf; + +// ============================================================================ +// Roundtrip harness (test-case-agnostic) +// ============================================================================ + +/// Result of a roundtrip conversion through a provider. +#[derive(Debug)] +struct RoundtripResult { + /// The provider JSON produced by `request_from_universal` + provider_json: Value, + /// The universal request recovered by `request_to_universal` + roundtripped: UniversalRequest, +} + +/// Harness for testing roundtrip conversions. Operates on any `UniversalRequest`. +struct RoundtripHarness; + +impl RoundtripHarness { + /// Convert a provider-native JSON payload to Universal first, then roundtrip + /// through the same or different providers. + fn from_provider_json( + source_format: ProviderFormat, + payload: Value, + ) -> Result { + let adapter = adapter_for_format(source_format) + .ok_or_else(|| format!("No adapter for {:?}", source_format))?; + adapter + .request_to_universal(payload) + .map_err(|e| format!("request_to_universal({:?}): {}", source_format, e)) + } + + /// Convert Universal -> Provider -> Universal and return both intermediate + result. + fn self_roundtrip( + req: &UniversalRequest, + format: ProviderFormat, + ) -> Result { + let adapter = + adapter_for_format(format).ok_or_else(|| format!("No adapter for {:?}", format))?; + + let mut req = req.clone(); + if req.model.is_none() { + req.model = Some(default_model(format).to_string()); + } + adapter.apply_defaults(&mut req); + + let provider_json = adapter + .request_from_universal(&req) + .map_err(|e| format!("request_from_universal({:?}): {}", format, e))?; + + let roundtripped = adapter + .request_to_universal(provider_json.clone()) + .map_err(|e| format!("request_to_universal({:?}): {}", format, e))?; + + Ok(RoundtripResult { + provider_json, + roundtripped, + }) + } + + /// Convert Universal -> Source -> Universal -> Target -> Universal. + fn cross_provider( + req: &UniversalRequest, + source: ProviderFormat, + target: ProviderFormat, + ) -> Result { + let source_result = Self::self_roundtrip(req, source)?; + + let target_adapter = + adapter_for_format(target).ok_or_else(|| format!("No adapter for {:?}", target))?; + + let mut universal_for_target = source_result.roundtripped; + universal_for_target.model = Some(default_model(target).to_string()); + target_adapter.apply_defaults(&mut universal_for_target); + + let target_json = target_adapter + .request_from_universal(&universal_for_target) + .map_err(|e| format!("request_from_universal({:?}): {}", target, e))?; + + let roundtripped = target_adapter + .request_to_universal(target_json.clone()) + .map_err(|e| format!("request_to_universal({:?}): {}", target, e))?; + + Ok(RoundtripResult { + provider_json: target_json, + roundtripped, + }) + } +} + +// ============================================================================ +// Content extraction (lossy-conversion-aware comparison) +// ============================================================================ + +/// A normalized per-message summary. Joins text so that `["a","b"]` and `["ab"]` +/// compare equal (providers may merge adjacent text blocks). Ignores `tool_name` +/// on tool results since OpenAI doesn't carry it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct MessageSummary { + role: &'static str, + text: String, + tool_call_ids: Vec, + tool_result_ids: Vec, +} + +fn summarize_messages(messages: &[Message]) -> Vec { + messages + .iter() + .map(|msg| match msg { + Message::System { content } => MessageSummary { + role: "system", + text: join_user_content(content), + tool_call_ids: vec![], + tool_result_ids: vec![], + }, + Message::User { content } => MessageSummary { + role: "user", + text: join_user_content(content), + tool_call_ids: vec![], + tool_result_ids: vec![], + }, + Message::Assistant { content, .. } => { + let (text, tool_call_ids) = summarize_assistant_content(content); + MessageSummary { + role: "assistant", + text, + tool_call_ids, + tool_result_ids: vec![], + } + } + Message::Tool { content } => MessageSummary { + role: "tool", + text: String::new(), + tool_call_ids: vec![], + tool_result_ids: content + .iter() + .map(|p| { + let ToolContentPart::ToolResult(tr) = p; + tr.tool_call_id.clone() + }) + .collect(), + }, + }) + .collect() +} + +fn join_user_content(content: &UserContent) -> String { + match content { + UserContent::String(s) => s.clone(), + UserContent::Array(parts) => parts + .iter() + .filter_map(|p| match p { + UserContentPart::Text(t) => Some(t.text.as_str()), + _ => None, + }) + .collect::>() + .join(""), + } +} + +fn summarize_assistant_content(content: &AssistantContent) -> (String, Vec) { + match content { + AssistantContent::String(s) => (s.clone(), vec![]), + AssistantContent::Array(parts) => { + let mut text = String::new(); + let mut tool_ids = Vec::new(); + for p in parts { + match p { + AssistantContentPart::Text(t) => text.push_str(&t.text), + AssistantContentPart::ToolCall { tool_call_id, .. } => { + tool_ids.push(tool_call_id.clone()); + } + AssistantContentPart::Reasoning { text: r, .. } => { + text.push_str(r); + } + _ => {} + } + } + (text, tool_ids) + } + } +} + +fn non_system(summaries: &[MessageSummary]) -> Vec<&MessageSummary> { + summaries.iter().filter(|s| s.role != "system").collect() +} + +fn extract_system_text(messages: &[Message]) -> String { + let summaries = summarize_messages(messages); + summaries + .iter() + .filter(|s| s.role == "system") + .map(|s| s.text.as_str()) + .collect::>() + .join("\n") +} + +fn default_model(format: ProviderFormat) -> &'static str { + match format { + ProviderFormat::OpenAI => "gpt-4", + ProviderFormat::Anthropic => "claude-3-5-sonnet-20241022", + ProviderFormat::Google => "gemini-1.5-flash", + ProviderFormat::Responses => "gpt-4", + _ => "test-model", + } +} + +fn parse_format(s: &str) -> ProviderFormat { + match s { + "openai" => ProviderFormat::OpenAI, + "anthropic" => ProviderFormat::Anthropic, + "google" => ProviderFormat::Google, + "responses" => ProviderFormat::Responses, + other => panic!("Unknown format in test case: {:?}", other), + } +} + +// ============================================================================ +// Assertion helpers +// ============================================================================ + +fn assert_anthropic_system_preserved(original: &UniversalRequest, anthropic_json: &Value) { + let original_system = extract_system_text(&original.messages); + if original_system.is_empty() { + return; + } + + let system_field = anthropic_json.get("system"); + assert!( + system_field.is_some(), + "Anthropic JSON should have 'system' field.\n\ + Original system text: {:?}\n\ + Anthropic JSON: {}", + original_system, + serde_json::to_string_pretty(anthropic_json).unwrap_or_default() + ); + + let system_text = system_field.unwrap().as_str().unwrap_or(""); + assert!( + !system_text.is_empty(), + "Anthropic 'system' field should not be empty.\n\ + Original system text: {:?}\n\ + Anthropic JSON: {}", + original_system, + serde_json::to_string_pretty(anthropic_json).unwrap_or_default() + ); +} + +fn assert_messages_preserved( + original: &UniversalRequest, + roundtripped: &UniversalRequest, + context: &str, +) { + let orig = summarize_messages(&original.messages); + let rt = summarize_messages(&roundtripped.messages); + + assert_eq!( + non_system(&orig), + non_system(&rt), + "Non-system messages should be preserved in {}.\n\ + Original: {:#?}\n\ + Roundtripped: {:#?}", + context, + non_system(&orig), + non_system(&rt) + ); +} + +// ============================================================================ +// Saved test case loading (roundtrip_cases.json) +// ============================================================================ + +#[derive(serde::Deserialize, Debug)] +struct SavedTestCase { + description: String, + source_format: String, + payload: Value, + /// Provider formats to roundtrip through. Each one runs: + /// source -> Universal -> target -> Universal + roundtrip_through: Vec, +} + +fn load_saved_cases() -> Vec { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/roundtrip_cases.json"); + let content = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("Failed to read {}: {}", path.display(), e)); + serde_json::from_str(&content) + .unwrap_or_else(|e| panic!("Failed to parse {}: {}", path.display(), e)) +} + +/// When proptest finds a failure, call this to get a JSON snippet you can paste +/// into `roundtrip_cases.json`. +fn format_as_saved_case( + description: &str, + source_format: ProviderFormat, + provider_json: &Value, + roundtrip_through: &[&str], +) -> String { + let case = json!({ + "description": description, + "source_format": format!("{}", source_format).to_lowercase(), + "payload": provider_json, + "roundtrip_through": roundtrip_through, + }); + serde_json::to_string_pretty(&case).unwrap() +} + +// ============================================================================ +// Saved case tests +// ============================================================================ + +#[test] +fn saved_cases_self_roundtrip() { + let cases = load_saved_cases(); + for case in &cases { + let source_format = parse_format(&case.source_format); + + // Parse from source format to Universal + let universal = RoundtripHarness::from_provider_json(source_format, case.payload.clone()) + .unwrap_or_else(|e| { + panic!( + "[{}] Failed to parse {} payload: {}", + case.description, case.source_format, e + ) + }); + + // Self-roundtrip through source format + let result = + RoundtripHarness::self_roundtrip(&universal, source_format).unwrap_or_else(|e| { + panic!( + "[{}] Self-roundtrip through {} failed: {}", + case.description, case.source_format, e + ) + }); + + assert_messages_preserved( + &universal, + &result.roundtripped, + &format!( + "[{}] self-roundtrip through {}", + case.description, case.source_format + ), + ); + + // For Anthropic, also check system preservation + if source_format == ProviderFormat::Anthropic { + assert_anthropic_system_preserved(&universal, &result.provider_json); + } + } +} + +#[test] +fn saved_cases_cross_provider() { + let cases = load_saved_cases(); + for case in &cases { + let source_format = parse_format(&case.source_format); + + let universal = RoundtripHarness::from_provider_json(source_format, case.payload.clone()) + .unwrap_or_else(|e| { + panic!( + "[{}] Failed to parse {} payload: {}", + case.description, case.source_format, e + ) + }); + + for target_name in &case.roundtrip_through { + let target_format = parse_format(target_name); + + let result = RoundtripHarness::cross_provider(&universal, source_format, target_format) + .unwrap_or_else(|e| { + panic!( + "[{}] {} -> {} failed: {}", + case.description, case.source_format, target_name, e + ) + }); + + assert_messages_preserved( + &universal, + &result.roundtripped, + &format!( + "[{}] {} -> {}", + case.description, case.source_format, target_name + ), + ); + + // For Anthropic targets, verify system not dropped + if target_format == ProviderFormat::Anthropic { + assert_anthropic_system_preserved(&universal, &result.provider_json); + } + } + } +} + +// ============================================================================ +// Proptest strategies +// ============================================================================ + +mod strategies { + use super::*; + use proptest::prelude::*; + + pub fn arb_text() -> impl Strategy { + "[a-zA-Z0-9 .!?,]{1,80}" + } + + fn arb_text_part() -> impl Strategy { + arb_text().prop_map(|text| TextContentPart { + text, + provider_options: None, + }) + } + + pub fn arb_user_content() -> impl Strategy { + prop_oneof![ + arb_text().prop_map(UserContent::String), + proptest::collection::vec(arb_text_part().prop_map(UserContentPart::Text), 1..=3) + .prop_map(UserContent::Array), + ] + } + + fn arb_tool_call_arguments() -> impl Strategy { + prop_oneof![ + proptest::collection::hash_map("[a-z]{2,8}", arb_json_value(), 0..=3).prop_map(|m| { + let map: serde_json::Map = m.into_iter().collect(); + ToolCallArguments::Valid(map) + }), + "[^{}]{1,30}".prop_map(ToolCallArguments::Invalid), + ] + } + + fn arb_json_value() -> impl Strategy { + prop_oneof![ + Just(Value::Null), + any::().prop_map(Value::Bool), + any::().prop_map(|i| Value::Number(i.into())), + arb_text().prop_map(Value::String), + ] + } + + fn arb_assistant_content() -> impl Strategy { + prop_oneof![ + 3 => arb_text().prop_map(AssistantContent::String), + 1 => proptest::collection::vec( + prop_oneof![ + 3 => arb_text_part().prop_map(AssistantContentPart::Text), + 1 => ( + "call_[a-zA-Z0-9]{8}", + "[a-z_]{3,12}", + arb_tool_call_arguments(), + ).prop_map(|(id, name, args)| AssistantContentPart::ToolCall { + tool_call_id: id, + tool_name: name, + arguments: args, + provider_options: None, + provider_executed: None, + }), + ], + 1..=3, + ).prop_map(AssistantContent::Array), + ] + } + + fn arb_tool_content() -> impl Strategy { + proptest::collection::vec( + ("call_[a-zA-Z0-9]{8}", "[a-z_]{3,12}", arb_json_value()).prop_map( + |(id, name, output)| { + ToolContentPart::ToolResult(ToolResultContentPart { + tool_call_id: id, + tool_name: name, + output, + provider_options: None, + }) + }, + ), + 1..=2, + ) + } + + pub fn arb_message() -> impl Strategy { + prop_oneof![ + 2 => arb_user_content().prop_map(|c| Message::System { content: c }), + 3 => arb_user_content().prop_map(|c| Message::User { content: c }), + 3 => arb_assistant_content().prop_map(|c| Message::Assistant { + content: c, + id: None, + }), + 1 => arb_tool_content().prop_map(|c| Message::Tool { content: c }), + ] + } + + /// Generate a valid-ish message thread (doesn't enforce strict role alternation). + pub fn arb_message_thread() -> impl Strategy> { + proptest::collection::vec(arb_message(), 1..=8) + } + + pub fn arb_universal_request() -> impl Strategy { + ( + arb_message_thread(), + prop::option::of(0.0..=2.0_f64), + prop::option::of(1i64..=8192), + ) + .prop_map(|(messages, temperature, max_tokens)| UniversalRequest { + model: None, + messages, + params: UniversalParams { + temperature, + max_tokens, + ..Default::default() + }, + }) + } +} + +// ============================================================================ +// Proptest-driven tests +// ============================================================================ + +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(256))] + + /// OpenAI self-roundtrip: Universal -> OpenAI -> Universal preserves non-system messages. + #[test] + fn prop_openai_self_roundtrip(req in strategies::arb_universal_request()) { + if let Ok(result) = RoundtripHarness::self_roundtrip(&req, ProviderFormat::OpenAI) { + let orig = summarize_messages(&req.messages); + let rt = summarize_messages(&result.roundtripped.messages); + prop_assert_eq!( + non_system(&orig), non_system(&rt), + "OpenAI self-roundtrip lost messages.\n\ + To save as a test case, add to roundtrip_cases.json:\n{}", + format_as_saved_case( + "proptest: OpenAI roundtrip failure", + ProviderFormat::OpenAI, + &result.provider_json, + &["openai"], + ) + ); + } + } + + /// Anthropic self-roundtrip: system messages must not be silently dropped. + #[test] + fn prop_anthropic_self_roundtrip(req in strategies::arb_universal_request()) { + if let Ok(result) = RoundtripHarness::self_roundtrip(&req, ProviderFormat::Anthropic) { + let system_text = extract_system_text(&req.messages); + if !system_text.is_empty() { + let system_field = result.provider_json.get("system"); + let saved = format_as_saved_case( + "proptest: Anthropic system dropped", + ProviderFormat::Anthropic, + &result.provider_json, + &["anthropic"], + ); + prop_assert!( + system_field.is_some(), + "Anthropic should have 'system' field. Original: {:?}\n\ + To save:\n{}", system_text, saved + ); + let field_text = system_field.unwrap().as_str().unwrap_or(""); + prop_assert!( + !field_text.is_empty(), + "Anthropic 'system' should not be empty. Original: {:?}\n\ + To save:\n{}", system_text, saved + ); + } + + let orig = summarize_messages(&req.messages); + let rt = summarize_messages(&result.roundtripped.messages); + prop_assert_eq!( + non_system(&orig), non_system(&rt), + "Anthropic self-roundtrip lost non-system messages.\n\ + To save:\n{}", + format_as_saved_case( + "proptest: Anthropic roundtrip failure", + ProviderFormat::Anthropic, + &result.provider_json, + &["anthropic"], + ) + ); + } + } + + /// Cross-provider: OpenAI -> Anthropic preserves non-system message content. + #[test] + fn prop_openai_to_anthropic(req in strategies::arb_universal_request()) { + if let Ok(result) = RoundtripHarness::cross_provider( + &req, ProviderFormat::OpenAI, ProviderFormat::Anthropic, + ) { + let orig = summarize_messages(&req.messages); + let rt = summarize_messages(&result.roundtripped.messages); + prop_assert_eq!( + non_system(&orig), non_system(&rt), + "OpenAI->Anthropic lost non-system messages.\n\ + To save:\n{}", + format_as_saved_case( + "proptest: OpenAI->Anthropic failure", + ProviderFormat::Anthropic, + &result.provider_json, + &["anthropic", "openai"], + ) + ); + } + } + + /// Cross-provider: Anthropic -> OpenAI preserves non-system message content. + #[test] + fn prop_anthropic_to_openai(req in strategies::arb_universal_request()) { + if let Ok(result) = RoundtripHarness::cross_provider( + &req, ProviderFormat::Anthropic, ProviderFormat::OpenAI, + ) { + let orig = summarize_messages(&req.messages); + let rt = summarize_messages(&result.roundtripped.messages); + prop_assert_eq!( + non_system(&orig), non_system(&rt), + "Anthropic->OpenAI lost non-system messages.\n\ + To save:\n{}", + format_as_saved_case( + "proptest: Anthropic->OpenAI failure", + ProviderFormat::OpenAI, + &result.provider_json, + &["openai", "anthropic"], + ) + ); + } + } + + /// Targeted: random UserContent variants for system messages are never dropped by Anthropic. + #[test] + fn prop_system_message_variants_not_dropped( + content in strategies::arb_user_content() + ) { + let req = UniversalRequest { + model: Some("claude-3-5-sonnet-20241022".into()), + messages: vec![ + Message::System { content }, + Message::User { + content: UserContent::String("Hello".into()), + }, + ], + params: UniversalParams { + max_tokens: Some(1024), + ..Default::default() + }, + }; + + let result = RoundtripHarness::self_roundtrip(&req, ProviderFormat::Anthropic).unwrap(); + let system_text = extract_system_text(&req.messages); + let saved = format_as_saved_case( + "proptest: system variant dropped", + ProviderFormat::Anthropic, + &result.provider_json, + &["anthropic"], + ); + prop_assert!( + result.provider_json.get("system").is_some(), + "Anthropic should have 'system' field. Original: {:?}\nTo save:\n{}", system_text, saved + ); + let field_text = result.provider_json.get("system").unwrap().as_str().unwrap_or(""); + prop_assert!( + !field_text.is_empty(), + "Anthropic 'system' should not be empty. Original: {:?}\nTo save:\n{}", system_text, saved + ); + } +} diff --git a/crates/lingua/tests/roundtrip_cases.json b/crates/lingua/tests/roundtrip_cases.json new file mode 100644 index 0000000..79d8d14 --- /dev/null +++ b/crates/lingua/tests/roundtrip_cases.json @@ -0,0 +1,159 @@ +[ + { + "description": "PR #83 regression: system message with array content (OpenAI format)", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Always be concise."} + ] + }, + { + "role": "user", + "content": "Hello" + } + ] + }, + "roundtrip_through": ["openai", "anthropic", "google"] + }, + { + "description": "PR #83 regression: system message with string content (baseline)", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello" + } + ] + }, + "roundtrip_through": ["openai", "anthropic", "google"] + }, + { + "description": "Multi-turn conversation with tool calls (OpenAI format)", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What is the weather in San Francisco?" + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\":\"San Francisco\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": "{\"temperature\":65,\"unit\":\"F\"}" + } + ] + }, + "roundtrip_through": ["openai", "anthropic"] + }, + { + "description": "Anthropic native format with system + multi-turn", + "source_format": "anthropic", + "payload": { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 1024, + "system": "You are a helpful assistant.", + "messages": [ + { + "role": "user", + "content": "Hello" + }, + { + "role": "assistant", + "content": "Hi there! How can I help?" + }, + { + "role": "user", + "content": "Tell me a joke." + } + ] + }, + "roundtrip_through": ["anthropic", "openai"] + }, + { + "description": "Anthropic native format with array user content", + "source_format": "anthropic", + "payload": { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello, how are you?"} + ] + } + ] + }, + "roundtrip_through": ["anthropic", "openai"] + }, + { + "description": "Google native format with system instruction", + "source_format": "google", + "payload": { + "model": "gemini-1.5-flash", + "systemInstruction": { + "parts": [ + {"text": "You are a helpful assistant."} + ] + }, + "contents": [ + { + "role": "user", + "parts": [ + {"text": "Hello"} + ] + } + ], + "generationConfig": { + "maxOutputTokens": 1024 + } + }, + "roundtrip_through": ["google", "openai", "anthropic"] + }, + { + "description": "OpenAI multi-part user content", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Part one."}, + {"type": "text", "text": "Part two."} + ] + } + ] + }, + "roundtrip_through": ["openai", "anthropic"] + } +] From d4d67d1a8fb4c3eaf98ef5599dfef9ce43344a33 Mon Sep 17 00:00:00 2001 From: Ankur Goyal Date: Fri, 6 Feb 2026 00:05:16 -0800 Subject: [PATCH 2/2] add tests --- .../tests/fuzz_roundtrip.proptest-regressions | 11 - crates/lingua/tests/fuzz_roundtrip.rs | 433 +++++++++--------- crates/lingua/tests/roundtrip_cases.json | 87 ++++ crates/lingua/tests/schema_strategy.rs | 335 ++++++++++++++ 4 files changed, 644 insertions(+), 222 deletions(-) delete mode 100644 crates/lingua/tests/fuzz_roundtrip.proptest-regressions create mode 100644 crates/lingua/tests/schema_strategy.rs diff --git a/crates/lingua/tests/fuzz_roundtrip.proptest-regressions b/crates/lingua/tests/fuzz_roundtrip.proptest-regressions deleted file mode 100644 index 0941aec..0000000 --- a/crates/lingua/tests/fuzz_roundtrip.proptest-regressions +++ /dev/null @@ -1,11 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 571bebd27a681a4afddc88a50ab68906c2411c4653316c807667968e1da77dbf # shrinks to content = Array([Text(TextContentPart { text: "a", provider_options: None })]) -cc 9c78a0e43216173390bc5704b23d8c3f03e950c41b292798b57e71ffa7b3e460 # shrinks to req = UniversalRequest { model: None, messages: [System { content: Array([Text(TextContentPart { text: "0", provider_options: None })]) }], params: UniversalParams { temperature: None, top_p: None, top_k: None, seed: None, presence_penalty: None, frequency_penalty: None, max_tokens: None, stop: None, logprobs: None, top_logprobs: None, tools: None, tool_choice: None, parallel_tool_calls: None, response_format: None, reasoning: None, metadata: None, store: None, service_tier: None, stream: None, extras: {} } } -cc f4f66bfaf60b020ad8dd448344f0127d0ffbf59ce9d042ee4755eb43d1de9131 # shrinks to req = UniversalRequest { model: None, messages: [Tool { content: [ToolResult(ToolResultContentPart { tool_call_id: "call_0aA00a00", tool_name: "___", output: Null, provider_options: None }), ToolResult(ToolResultContentPart { tool_call_id: "call_A0aaaaAA", tool_name: "aaa", output: Null, provider_options: None })] }], params: UniversalParams { temperature: None, top_p: None, top_k: None, seed: None, presence_penalty: None, frequency_penalty: None, max_tokens: None, stop: None, logprobs: None, top_logprobs: None, tools: None, tool_choice: None, parallel_tool_calls: None, response_format: None, reasoning: None, metadata: None, store: None, service_tier: None, stream: None, extras: {} } } -cc 3b06f6ee7178d80cdcf18873d071b554ef516e474543d66deee9c2bc3e2dcaec # shrinks to req = UniversalRequest { model: None, messages: [Tool { content: [ToolResult(ToolResultContentPart { tool_call_id: "call_0aaAAAA0", tool_name: "aa_", output: Null, provider_options: None }), ToolResult(ToolResultContentPart { tool_call_id: "call_0aaAAAaa", tool_name: "___", output: Null, provider_options: None })] }], params: UniversalParams { temperature: None, top_p: None, top_k: None, seed: None, presence_penalty: None, frequency_penalty: None, max_tokens: None, stop: None, logprobs: None, top_logprobs: None, tools: None, tool_choice: None, parallel_tool_calls: None, response_format: None, reasoning: None, metadata: None, store: None, service_tier: None, stream: None, extras: {} } } -cc 992d77a84f2c6777ee72c0183797d3015725b3d04feee90c992a94d4f5478aea # shrinks to req = UniversalRequest { model: None, messages: [Tool { content: [ToolResult(ToolResultContentPart { tool_call_id: "call_a00Aaa0a", tool_name: "a_a", output: Null, provider_options: None }), ToolResult(ToolResultContentPart { tool_call_id: "call_A0Aa00a0", tool_name: "_a_", output: Null, provider_options: None })] }], params: UniversalParams { temperature: None, top_p: None, top_k: None, seed: None, presence_penalty: None, frequency_penalty: None, max_tokens: None, stop: None, logprobs: None, top_logprobs: None, tools: None, tool_choice: None, parallel_tool_calls: None, response_format: None, reasoning: None, metadata: None, store: None, service_tier: None, stream: None, extras: {} } } diff --git a/crates/lingua/tests/fuzz_roundtrip.rs b/crates/lingua/tests/fuzz_roundtrip.rs index ba57dd9..10a6353 100644 --- a/crates/lingua/tests/fuzz_roundtrip.rs +++ b/crates/lingua/tests/fuzz_roundtrip.rs @@ -12,9 +12,11 @@ use lingua::processing::adapter_for_format; use lingua::serde_json::{self, json, Value}; use lingua::universal::message::*; -use lingua::{ProviderFormat, UniversalParams, UniversalRequest}; +use lingua::{ProviderFormat, UniversalRequest}; use std::path::PathBuf; +mod schema_strategy; + // ============================================================================ // Roundtrip harness (test-case-agnostic) // ============================================================================ @@ -407,123 +409,135 @@ fn saved_cases_cross_provider() { } // ============================================================================ -// Proptest strategies +// Proptest strategies: generate provider-native JSON payloads // ============================================================================ mod strategies { + use super::schema_strategy::{load_openapi_definitions, strategy_for_schema_name}; use super::*; use proptest::prelude::*; + // -- Shared primitives -- + pub fn arb_text() -> impl Strategy { "[a-zA-Z0-9 .!?,]{1,80}" } - fn arb_text_part() -> impl Strategy { - arb_text().prop_map(|text| TextContentPart { - text, - provider_options: None, - }) - } - - pub fn arb_user_content() -> impl Strategy { - prop_oneof![ - arb_text().prop_map(UserContent::String), - proptest::collection::vec(arb_text_part().prop_map(UserContentPart::Text), 1..=3) - .prop_map(UserContent::Array), - ] - } - - fn arb_tool_call_arguments() -> impl Strategy { - prop_oneof![ - proptest::collection::hash_map("[a-z]{2,8}", arb_json_value(), 0..=3).prop_map(|m| { - let map: serde_json::Map = m.into_iter().collect(); - ToolCallArguments::Valid(map) - }), - "[^{}]{1,30}".prop_map(ToolCallArguments::Invalid), - ] - } - fn arb_json_value() -> impl Strategy { prop_oneof![ Just(Value::Null), any::().prop_map(Value::Bool), - any::().prop_map(|i| Value::Number(i.into())), + any::().prop_map(|i| json!(i)), arb_text().prop_map(Value::String), ] } - fn arb_assistant_content() -> impl Strategy { - prop_oneof![ - 3 => arb_text().prop_map(AssistantContent::String), - 1 => proptest::collection::vec( - prop_oneof![ - 3 => arb_text_part().prop_map(AssistantContentPart::Text), - 1 => ( - "call_[a-zA-Z0-9]{8}", - "[a-z_]{3,12}", - arb_tool_call_arguments(), - ).prop_map(|(id, name, args)| AssistantContentPart::ToolCall { - tool_call_id: id, - tool_name: name, - arguments: args, - provider_options: None, - provider_executed: None, - }), - ], - 1..=3, - ).prop_map(AssistantContent::Array), - ] + fn arb_json_object() -> impl Strategy { + proptest::collection::hash_map("[a-z]{2,8}", arb_json_value(), 0..=3) + .prop_map(|m| json!(m.into_iter().collect::>())) } - fn arb_tool_content() -> impl Strategy { - proptest::collection::vec( - ("call_[a-zA-Z0-9]{8}", "[a-z_]{3,12}", arb_json_value()).prop_map( - |(id, name, output)| { - ToolContentPart::ToolResult(ToolResultContentPart { - tool_call_id: id, - tool_name: name, - output, - provider_options: None, - }) - }, - ), - 1..=2, + fn arb_function_name() -> impl Strategy { + "[a-z_]{3,12}" + } + + // ======================================================================== + // Schema-driven strategies (OpenAI + Anthropic from OpenAPI specs) + // ======================================================================== + + fn specs_dir() -> String { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + // specs/ is at the repo root, two levels up from crates/lingua/ + format!("{}/../..", manifest_dir) + } + + pub fn arb_openai_payload() -> BoxedStrategy { + let defs = load_openapi_definitions(&format!("{}/specs/openai/openapi.yml", specs_dir())); + strategy_for_schema_name("CreateChatCompletionRequest", &defs) + } + + pub fn arb_anthropic_payload() -> BoxedStrategy { + let defs = + load_openapi_definitions(&format!("{}/specs/anthropic/openapi.yml", specs_dir())); + strategy_for_schema_name("CreateMessageParams", &defs) + } + + // ======================================================================== + // Google GenerateContent JSON strategies (hand-written, no OpenAPI spec) + // ======================================================================== + + fn google_text_part() -> impl Strategy { + arb_text().prop_map(|t| json!({"text": t})) + } + + fn google_function_call_part() -> impl Strategy { + (arb_function_name(), arb_json_object()) + .prop_map(|(name, args)| json!({"functionCall": {"name": name, "args": args}})) + } + + fn google_function_response_part() -> impl Strategy { + (arb_function_name(), arb_json_object()).prop_map( + |(name, response)| json!({"functionResponse": {"name": name, "response": response}}), ) } - pub fn arb_message() -> impl Strategy { + fn google_user_content() -> impl Strategy { + proptest::collection::vec(google_text_part(), 1..=3) + .prop_map(|parts| json!({"role": "user", "parts": parts})) + } + + fn google_model_content() -> impl Strategy { prop_oneof![ - 2 => arb_user_content().prop_map(|c| Message::System { content: c }), - 3 => arb_user_content().prop_map(|c| Message::User { content: c }), - 3 => arb_assistant_content().prop_map(|c| Message::Assistant { - content: c, - id: None, - }), - 1 => arb_tool_content().prop_map(|c| Message::Tool { content: c }), + // Text-only + 3 => proptest::collection::vec(google_text_part(), 1..=3) + .prop_map(|parts| json!({"role": "model", "parts": parts})), + // Function calls + 1 => proptest::collection::vec(google_function_call_part(), 1..=2) + .prop_map(|parts| json!({"role": "model", "parts": parts})), ] } - /// Generate a valid-ish message thread (doesn't enforce strict role alternation). - pub fn arb_message_thread() -> impl Strategy> { - proptest::collection::vec(arb_message(), 1..=8) + fn google_tool_response_content() -> impl Strategy { + proptest::collection::vec(google_function_response_part(), 1..=2) + .prop_map(|parts| json!({"role": "user", "parts": parts})) } - pub fn arb_universal_request() -> impl Strategy { + fn google_content() -> impl Strategy { + prop_oneof![ + 3 => google_user_content(), + 3 => google_model_content(), + 1 => google_tool_response_content(), + ] + } + + pub fn arb_google_payload() -> impl Strategy { ( - arb_message_thread(), - prop::option::of(0.0..=2.0_f64), - prop::option::of(1i64..=8192), + proptest::collection::vec(google_content(), 1..=6), + prop::option::of(arb_text()), // optional systemInstruction ) - .prop_map(|(messages, temperature, max_tokens)| UniversalRequest { - model: None, - messages, - params: UniversalParams { - temperature, - max_tokens, - ..Default::default() - }, + .prop_map(|(contents, system)| { + let mut payload = json!({ + "contents": contents, + "generationConfig": {"maxOutputTokens": 1024}, + }); + if let Some(s) = system { + payload["systemInstruction"] = json!({"parts": [{"text": s}]}); + } + payload }) } + + // ======================================================================== + // Combined: generate (format, payload) pairs + // ======================================================================== + + pub fn arb_provider_payload() -> impl Strategy { + prop_oneof![ + arb_openai_payload().prop_map(|p| (ProviderFormat::OpenAI, p)), + arb_anthropic_payload().prop_map(|p| (ProviderFormat::Anthropic, p)), + arb_google_payload().prop_map(|p| (ProviderFormat::Google, p)), + ] + } } // ============================================================================ @@ -532,150 +546,147 @@ mod strategies { use proptest::prelude::*; -proptest! { - #![proptest_config(ProptestConfig::with_cases(256))] +/// Shared helper: parse provider JSON to Universal, roundtrip through each target, +/// and check assertions. Returns Err with saveable JSON on failure. +fn run_roundtrips( + source_format: ProviderFormat, + payload: &Value, + targets: &[ProviderFormat], +) -> Result<(), String> { + let universal = RoundtripHarness::from_provider_json(source_format, payload.clone())?; + + // Self-roundtrip + let self_result = RoundtripHarness::self_roundtrip(&universal, source_format)?; + let orig = summarize_messages(&universal.messages); + let rt = summarize_messages(&self_result.roundtripped.messages); + if non_system(&orig) != non_system(&rt) { + return Err(format!( + "Self-roundtrip through {:?} lost messages.\n\ + Original: {:#?}\n\ + Roundtripped: {:#?}\n\ + To save:\n{}", + source_format, + non_system(&orig), + non_system(&rt), + format_as_saved_case( + &format!("proptest: {:?} self-roundtrip failure", source_format), + source_format, + payload, + &[&format!("{}", source_format).to_lowercase()], + ), + )); + } - /// OpenAI self-roundtrip: Universal -> OpenAI -> Universal preserves non-system messages. - #[test] - fn prop_openai_self_roundtrip(req in strategies::arb_universal_request()) { - if let Ok(result) = RoundtripHarness::self_roundtrip(&req, ProviderFormat::OpenAI) { - let orig = summarize_messages(&req.messages); + // Cross-provider roundtrips + for &target in targets { + if let Ok(result) = RoundtripHarness::cross_provider(&universal, source_format, target) { let rt = summarize_messages(&result.roundtripped.messages); - prop_assert_eq!( - non_system(&orig), non_system(&rt), - "OpenAI self-roundtrip lost messages.\n\ - To save as a test case, add to roundtrip_cases.json:\n{}", - format_as_saved_case( - "proptest: OpenAI roundtrip failure", - ProviderFormat::OpenAI, - &result.provider_json, - &["openai"], - ) - ); + if non_system(&orig) != non_system(&rt) { + return Err(format!( + "{:?} -> {:?} lost messages.\n\ + Original: {:#?}\n\ + Roundtripped: {:#?}\n\ + To save:\n{}", + source_format, + target, + non_system(&orig), + non_system(&rt), + format_as_saved_case( + &format!("proptest: {:?}->{:?} failure", source_format, target), + source_format, + payload, + &[ + &format!("{}", source_format).to_lowercase(), + &format!("{}", target).to_lowercase(), + ], + ), + )); + } + + if target == ProviderFormat::Anthropic { + let sys = extract_system_text(&universal.messages); + if !sys.is_empty() { + let s = result.provider_json.get("system"); + if s.is_none() || s.unwrap().as_str().unwrap_or("").is_empty() { + return Err(format!( + "Anthropic system dropped! Original: {:?}\nTo save:\n{}", + sys, + format_as_saved_case( + "proptest: Anthropic system dropped", + source_format, + payload, + &[&format!("{}", source_format).to_lowercase(), "anthropic",], + ), + )); + } + } + } } } - /// Anthropic self-roundtrip: system messages must not be silently dropped. + Ok(()) +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: 256, + // Don't write .proptest-regressions files; we use roundtrip_cases.json instead. + failure_persistence: None, + ..ProptestConfig::default() + })] + + /// OpenAI Chat Completions JSON -> roundtrip through all providers. #[test] - fn prop_anthropic_self_roundtrip(req in strategies::arb_universal_request()) { - if let Ok(result) = RoundtripHarness::self_roundtrip(&req, ProviderFormat::Anthropic) { - let system_text = extract_system_text(&req.messages); - if !system_text.is_empty() { - let system_field = result.provider_json.get("system"); - let saved = format_as_saved_case( - "proptest: Anthropic system dropped", - ProviderFormat::Anthropic, - &result.provider_json, - &["anthropic"], - ); - prop_assert!( - system_field.is_some(), - "Anthropic should have 'system' field. Original: {:?}\n\ - To save:\n{}", system_text, saved - ); - let field_text = system_field.unwrap().as_str().unwrap_or(""); - prop_assert!( - !field_text.is_empty(), - "Anthropic 'system' should not be empty. Original: {:?}\n\ - To save:\n{}", system_text, saved - ); + fn prop_openai_payload(payload in strategies::arb_openai_payload()) { + if let Ok(universal) = RoundtripHarness::from_provider_json(ProviderFormat::OpenAI, payload.clone()) { + let _ = universal; // parsed ok + if let Err(e) = run_roundtrips( + ProviderFormat::OpenAI, &payload, + &[ProviderFormat::Anthropic, ProviderFormat::Google], + ) { + prop_assert!(false, "{}", e); } - - let orig = summarize_messages(&req.messages); - let rt = summarize_messages(&result.roundtripped.messages); - prop_assert_eq!( - non_system(&orig), non_system(&rt), - "Anthropic self-roundtrip lost non-system messages.\n\ - To save:\n{}", - format_as_saved_case( - "proptest: Anthropic roundtrip failure", - ProviderFormat::Anthropic, - &result.provider_json, - &["anthropic"], - ) - ); } } - /// Cross-provider: OpenAI -> Anthropic preserves non-system message content. + /// Anthropic Messages JSON -> roundtrip through all providers. #[test] - fn prop_openai_to_anthropic(req in strategies::arb_universal_request()) { - if let Ok(result) = RoundtripHarness::cross_provider( - &req, ProviderFormat::OpenAI, ProviderFormat::Anthropic, - ) { - let orig = summarize_messages(&req.messages); - let rt = summarize_messages(&result.roundtripped.messages); - prop_assert_eq!( - non_system(&orig), non_system(&rt), - "OpenAI->Anthropic lost non-system messages.\n\ - To save:\n{}", - format_as_saved_case( - "proptest: OpenAI->Anthropic failure", - ProviderFormat::Anthropic, - &result.provider_json, - &["anthropic", "openai"], - ) - ); + fn prop_anthropic_payload(payload in strategies::arb_anthropic_payload()) { + if let Ok(universal) = RoundtripHarness::from_provider_json(ProviderFormat::Anthropic, payload.clone()) { + let _ = universal; + if let Err(e) = run_roundtrips( + ProviderFormat::Anthropic, &payload, + &[ProviderFormat::OpenAI, ProviderFormat::Google], + ) { + prop_assert!(false, "{}", e); + } } } - /// Cross-provider: Anthropic -> OpenAI preserves non-system message content. + /// Google GenerateContent JSON -> roundtrip through all providers. #[test] - fn prop_anthropic_to_openai(req in strategies::arb_universal_request()) { - if let Ok(result) = RoundtripHarness::cross_provider( - &req, ProviderFormat::Anthropic, ProviderFormat::OpenAI, - ) { - let orig = summarize_messages(&req.messages); - let rt = summarize_messages(&result.roundtripped.messages); - prop_assert_eq!( - non_system(&orig), non_system(&rt), - "Anthropic->OpenAI lost non-system messages.\n\ - To save:\n{}", - format_as_saved_case( - "proptest: Anthropic->OpenAI failure", - ProviderFormat::OpenAI, - &result.provider_json, - &["openai", "anthropic"], - ) - ); + fn prop_google_payload(payload in strategies::arb_google_payload()) { + if let Ok(universal) = RoundtripHarness::from_provider_json(ProviderFormat::Google, payload.clone()) { + let _ = universal; + if let Err(e) = run_roundtrips( + ProviderFormat::Google, &payload, + &[ProviderFormat::OpenAI, ProviderFormat::Anthropic], + ) { + prop_assert!(false, "{}", e); + } } } - /// Targeted: random UserContent variants for system messages are never dropped by Anthropic. + /// Any provider JSON -> all roundtrips (maximum coverage). #[test] - fn prop_system_message_variants_not_dropped( - content in strategies::arb_user_content() - ) { - let req = UniversalRequest { - model: Some("claude-3-5-sonnet-20241022".into()), - messages: vec![ - Message::System { content }, - Message::User { - content: UserContent::String("Hello".into()), - }, - ], - params: UniversalParams { - max_tokens: Some(1024), - ..Default::default() - }, - }; - - let result = RoundtripHarness::self_roundtrip(&req, ProviderFormat::Anthropic).unwrap(); - let system_text = extract_system_text(&req.messages); - let saved = format_as_saved_case( - "proptest: system variant dropped", - ProviderFormat::Anthropic, - &result.provider_json, - &["anthropic"], - ); - prop_assert!( - result.provider_json.get("system").is_some(), - "Anthropic should have 'system' field. Original: {:?}\nTo save:\n{}", system_text, saved - ); - let field_text = result.provider_json.get("system").unwrap().as_str().unwrap_or(""); - prop_assert!( - !field_text.is_empty(), - "Anthropic 'system' should not be empty. Original: {:?}\nTo save:\n{}", system_text, saved - ); + fn prop_any_provider((format, payload) in strategies::arb_provider_payload()) { + let all_targets = [ProviderFormat::OpenAI, ProviderFormat::Anthropic, ProviderFormat::Google]; + let targets: Vec<_> = all_targets.iter().copied().filter(|&t| t != format).collect(); + + if let Ok(_) = RoundtripHarness::from_provider_json(format, payload.clone()) { + if let Err(e) = run_roundtrips(format, &payload, &targets) { + prop_assert!(false, "{}", e); + } + } } } diff --git a/crates/lingua/tests/roundtrip_cases.json b/crates/lingua/tests/roundtrip_cases.json index 79d8d14..ca23a5c 100644 --- a/crates/lingua/tests/roundtrip_cases.json +++ b/crates/lingua/tests/roundtrip_cases.json @@ -155,5 +155,92 @@ ] }, "roundtrip_through": ["openai", "anthropic"] + }, + { + "description": "Developer message with array content -> Anthropic system dropped (schema-fuzz)", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "developer", + "content": [ + {"type": "text", "text": "You are a helpful assistant."} + ] + }, + { + "role": "user", + "content": "Hello" + } + ] + }, + "roundtrip_through": ["openai", "anthropic"] + }, + { + "description": "Anthropic thinking block text lost in cross-provider conversion (schema-fuzz)", + "source_format": "anthropic", + "payload": { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 1024, + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "Let me think about this carefully.", + "signature": "sig_abc123" + } + ] + } + ] + }, + "roundtrip_through": ["anthropic", "google"] + }, + { + "description": "Google consecutive same-role messages merged on roundtrip (schema-fuzz)", + "source_format": "google", + "payload": { + "contents": [ + { + "role": "user", + "parts": [{"text": "First message"}] + }, + { + "role": "user", + "parts": [{"text": "Second message"}] + } + ], + "generationConfig": {"maxOutputTokens": 1024} + }, + "roundtrip_through": ["google"] + }, + { + "description": "Google multiple functionResponse parts reduced to one in OpenAI (schema-fuzz)", + "source_format": "google", + "payload": { + "contents": [ + { + "role": "user", + "parts": [{"text": "Call two functions"}] + }, + { + "role": "model", + "parts": [ + {"functionCall": {"name": "get_weather", "args": {"city": "SF"}}}, + {"functionCall": {"name": "get_time", "args": {"city": "SF"}}} + ] + }, + { + "role": "user", + "parts": [ + {"functionResponse": {"name": "get_weather", "response": {"temp": 65}}}, + {"functionResponse": {"name": "get_time", "response": {"time": "3pm"}}} + ] + } + ], + "generationConfig": {"maxOutputTokens": 1024} + }, + "roundtrip_through": ["google", "openai"] } ] diff --git a/crates/lingua/tests/schema_strategy.rs b/crates/lingua/tests/schema_strategy.rs new file mode 100644 index 0000000..e1f0ff7 --- /dev/null +++ b/crates/lingua/tests/schema_strategy.rs @@ -0,0 +1,335 @@ +//! Generate random `serde_json::Value` from OpenAPI / JSON Schema definitions. +//! +//! This module walks a JSON Schema tree and produces a `proptest::Strategy` that +//! generates conforming JSON values. It handles `$ref`, `const`, `enum`, +//! `anyOf`/`oneOf`, objects (required + optional properties), arrays, and +//! primitive types. +//! +//! Used by `fuzz_roundtrip.rs` to auto-generate provider request payloads +//! directly from the OpenAPI specs in `specs/`. + +use lingua::serde_json::{self, json, Map, Value}; +use proptest::prelude::*; +use std::sync::Arc; + +/// Context for schema resolution (definitions map + depth tracking). +#[derive(Clone)] +pub struct SchemaCtx { + definitions: Arc>, +} + +impl SchemaCtx { + pub fn new(definitions: Map) -> Self { + Self { + definitions: Arc::new(definitions), + } + } + + /// Look up a schema name from a `$ref` string like `#/components/schemas/Foo`. + fn resolve_ref(&self, ref_str: &str) -> Option<&Value> { + let name = ref_str.rsplit('/').next()?; + self.definitions.get(name) + } +} + +const MAX_DEPTH: usize = 12; + +/// Build a proptest `Strategy` that generates random JSON conforming to `schema`. +pub fn strategy_from_schema(schema: &Value, ctx: &SchemaCtx, depth: usize) -> BoxedStrategy { + if depth > MAX_DEPTH { + return Just(Value::Null).boxed(); + } + + // Handle $ref + if let Some(ref_str) = schema.get("$ref").and_then(|v| v.as_str()) { + if let Some(resolved) = ctx.resolve_ref(ref_str) { + let resolved = resolved.clone(); + let ctx = ctx.clone(); + return strategy_from_schema(&resolved, &ctx, depth + 1); + } + return Just(Value::Null).boxed(); + } + + // Handle const + if let Some(const_val) = schema.get("const") { + return Just(const_val.clone()).boxed(); + } + + // Handle enum + if let Some(enum_vals) = schema.get("enum").and_then(|v| v.as_array()) { + if !enum_vals.is_empty() { + let vals: Vec = enum_vals.clone(); + return (0..vals.len()).prop_map(move |i| vals[i].clone()).boxed(); + } + } + + // Handle anyOf / oneOf (pick one variant) + if let Some(variants) = schema + .get("anyOf") + .or_else(|| schema.get("oneOf")) + .and_then(|v| v.as_array()) + { + if !variants.is_empty() { + let strategies: Vec> = variants + .iter() + .map(|v| strategy_from_schema(v, ctx, depth + 1)) + .collect(); + return proptest::strategy::Union::new(strategies).boxed(); + } + } + + // Handle allOf (merge all schemas - simplified: just use the first object-like one) + if let Some(all_of) = schema.get("allOf").and_then(|v| v.as_array()) { + // Merge all properties into a single object schema + let mut merged_props = Map::new(); + let mut merged_required = Vec::new(); + for sub in all_of { + if let Some(props) = sub.get("properties").and_then(|p| p.as_object()) { + for (k, v) in props { + merged_props.insert(k.clone(), v.clone()); + } + } + if let Some(req) = sub.get("required").and_then(|r| r.as_array()) { + for r in req { + if let Some(s) = r.as_str() { + merged_required.push(Value::String(s.to_string())); + } + } + } + // Also resolve $ref within allOf + if let Some(ref_str) = sub.get("$ref").and_then(|v| v.as_str()) { + if let Some(resolved) = ctx.resolve_ref(ref_str) { + if let Some(props) = resolved.get("properties").and_then(|p| p.as_object()) { + for (k, v) in props { + merged_props.insert(k.clone(), v.clone()); + } + } + if let Some(req) = resolved.get("required").and_then(|r| r.as_array()) { + for r in req { + if let Some(s) = r.as_str() { + merged_required.push(Value::String(s.to_string())); + } + } + } + } + } + } + if !merged_props.is_empty() { + let merged = json!({ + "type": "object", + "properties": Value::Object(merged_props), + "required": Value::Array(merged_required), + }); + return strategy_from_schema(&merged, ctx, depth + 1); + } + } + + // Handle by type + let type_str = schema.get("type").and_then(|v| v.as_str()).unwrap_or(""); + + match type_str { + "string" => arb_string(schema).boxed(), + "integer" => (-1000i64..1000).prop_map(|i| json!(i)).boxed(), + "number" => (-100.0f64..100.0).prop_map(|f| json!(f)).boxed(), + "boolean" => any::().prop_map(Value::Bool).boxed(), + "null" => Just(Value::Null).boxed(), + "object" => strategy_for_object(schema, ctx, depth), + "array" => strategy_for_array(schema, ctx, depth), + _ => { + // No type specified - could be a ref-only or untyped schema + // Try to infer from other properties + if schema.get("properties").is_some() { + strategy_for_object(schema, ctx, depth) + } else if schema.get("items").is_some() { + strategy_for_array(schema, ctx, depth) + } else { + // Fallback: random primitive + prop_oneof![ + "[a-zA-Z0-9 ]{1,30}".prop_map(Value::String), + Just(Value::Null), + ] + .boxed() + } + } + } +} + +fn arb_string(schema: &Value) -> impl Strategy { + // If there are enum values, use those + if let Some(enum_vals) = schema.get("enum").and_then(|v| v.as_array()) { + let vals: Vec = enum_vals + .iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect(); + if !vals.is_empty() { + return (0..vals.len()) + .prop_map(move |i| Value::String(vals[i].clone())) + .boxed(); + } + } + "[a-zA-Z0-9 .!?,]{1,50}".prop_map(Value::String).boxed() +} + +fn strategy_for_object(schema: &Value, ctx: &SchemaCtx, depth: usize) -> BoxedStrategy { + let properties = schema + .get("properties") + .and_then(|v| v.as_object()) + .cloned() + .unwrap_or_default(); + + let required: Vec = schema + .get("required") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + + if properties.is_empty() { + return Just(json!({})).boxed(); + } + + // Separate required and optional properties + let mut required_entries: Vec<(String, Value)> = Vec::new(); + let mut optional_entries: Vec<(String, Value)> = Vec::new(); + + for (key, prop_schema) in &properties { + if required.contains(key) { + required_entries.push((key.clone(), prop_schema.clone())); + } else { + optional_entries.push((key.clone(), prop_schema.clone())); + } + } + + let ctx_clone = ctx.clone(); + + // Build strategies for required properties + let required_strategies: Vec<(String, BoxedStrategy)> = required_entries + .into_iter() + .map(|(key, prop_schema)| { + let strat = strategy_from_schema(&prop_schema, &ctx_clone, depth + 1); + (key, strat) + }) + .collect(); + + // Build strategies for optional properties (each may or may not be included) + let optional_strategies: Vec<(String, BoxedStrategy>)> = optional_entries + .into_iter() + .map(|(key, prop_schema)| { + let strat = + prop::option::of(strategy_from_schema(&prop_schema, &ctx_clone, depth + 1)).boxed(); + (key, strat) + }) + .collect(); + + // We need to combine all strategies. Use a tuple approach with vectors. + // Generate required values as a vec, optional as a vec of options. + let req_strat = required_strategies + .into_iter() + .map(|(k, s)| s.prop_map(move |v| (k.clone(), v))) + .collect::>(); + + let opt_strat = optional_strategies + .into_iter() + .map(|(k, s)| s.prop_map(move |v| (k.clone(), v))) + .collect::>(); + + // Use prop_flat_map to combine dynamically-sized strategy lists + let req_values = if req_strat.is_empty() { + Just(Vec::<(String, Value)>::new()).boxed() + } else { + req_strat.into_iter().fold( + Just(Vec::<(String, Value)>::new()).boxed(), + |acc, item_strat| { + (acc, item_strat) + .prop_map(|(mut vec, item)| { + vec.push(item); + vec + }) + .boxed() + }, + ) + }; + + let opt_values = if opt_strat.is_empty() { + Just(Vec::<(String, Option)>::new()).boxed() + } else { + opt_strat.into_iter().fold( + Just(Vec::<(String, Option)>::new()).boxed(), + |acc, item_strat| { + (acc, item_strat) + .prop_map(|(mut vec, item)| { + vec.push(item); + vec + }) + .boxed() + }, + ) + }; + + (req_values, opt_values) + .prop_map(|(required_vals, optional_vals)| { + let mut map = Map::new(); + for (key, val) in required_vals { + map.insert(key, val); + } + for (key, maybe_val) in optional_vals { + if let Some(val) = maybe_val { + map.insert(key, val); + } + } + Value::Object(map) + }) + .boxed() +} + +fn strategy_for_array(schema: &Value, ctx: &SchemaCtx, depth: usize) -> BoxedStrategy { + let items_schema = schema.get("items").cloned().unwrap_or(json!({})); + let min_items = schema.get("minItems").and_then(|v| v.as_u64()).unwrap_or(1) as usize; + let max_items = schema + .get("maxItems") + .and_then(|v| v.as_u64()) + .unwrap_or(3) + .min(5) as usize; + let max_items = max_items.max(min_items); + + let item_strat = strategy_from_schema(&items_schema, ctx, depth + 1); + proptest::collection::vec(item_strat, min_items..=max_items) + .prop_map(Value::Array) + .boxed() +} + +// ============================================================================ +// Spec loading helpers +// ============================================================================ + +/// Load an OpenAPI spec and extract the `components.schemas` definitions map. +pub fn load_openapi_definitions(spec_path: &str) -> Map { + let content = std::fs::read_to_string(spec_path) + .unwrap_or_else(|e| panic!("Failed to read spec at {}: {}", spec_path, e)); + + // Try JSON first (Anthropic spec is JSON despite .yml extension), then YAML + let spec: Value = serde_json::from_str(&content) + .or_else(|_| serde_yaml::from_str::(&content).map_err(|e| e.to_string())) + .unwrap_or_else(|e| panic!("Failed to parse spec at {}: {}", spec_path, e)); + + spec.get("components") + .and_then(|c| c.get("schemas")) + .and_then(|s| s.as_object()) + .cloned() + .unwrap_or_else(|| panic!("No components.schemas in {}", spec_path)) +} + +/// Build a strategy for a named schema from the definitions map. +pub fn strategy_for_schema_name( + name: &str, + definitions: &Map, +) -> BoxedStrategy { + let schema = definitions + .get(name) + .unwrap_or_else(|| panic!("Schema '{}' not found in definitions", name)); + let ctx = SchemaCtx::new(definitions.clone()); + strategy_from_schema(schema, &ctx, 0) +}