diff --git a/Cargo.lock b/Cargo.lock index b3d1f4d..0b5192e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2080,6 +2080,7 @@ dependencies = [ "paste", "pbjson", "pbjson-types", + "proptest", "prost", "prost-types", "pyo3", diff --git a/crates/lingua/Cargo.toml b/crates/lingua/Cargo.toml index d9fa869..1567d59 100644 --- a/crates/lingua/Cargo.toml +++ b/crates/lingua/Cargo.toml @@ -58,6 +58,7 @@ tokio-test.workspace = true paste.workspace = true log.workspace = true env_logger.workspace = true +proptest.workspace = true [lib] name = "lingua" diff --git a/crates/lingua/tests/fuzz_roundtrip.rs b/crates/lingua/tests/fuzz_roundtrip.rs new file mode 100644 index 0000000..10a6353 --- /dev/null +++ b/crates/lingua/tests/fuzz_roundtrip.rs @@ -0,0 +1,692 @@ +//! Property-based roundtrip tests for Lingua cross-provider conversions. +//! +//! Test cases come from two sources: +//! 1. **Saved cases** (`roundtrip_cases.json`): Hand-written provider JSON payloads in any +//! format (OpenAI, Anthropic, Google, etc.). When proptest finds a novel failure, copy +//! the provider JSON into this file to make it a permanent regression test. +//! 2. **Proptest strategies**: Random `UniversalRequest` generation for coverage. +//! +//! The roundtrip harness is format-agnostic: it accepts either a `UniversalRequest` directly +//! or a `(ProviderFormat, serde_json::Value)` pair and runs the same assertions. + +use lingua::processing::adapter_for_format; +use lingua::serde_json::{self, json, Value}; +use lingua::universal::message::*; +use lingua::{ProviderFormat, UniversalRequest}; +use std::path::PathBuf; + +mod schema_strategy; + +// ============================================================================ +// Roundtrip harness (test-case-agnostic) +// ============================================================================ + +/// Result of a roundtrip conversion through a provider. +#[derive(Debug)] +struct RoundtripResult { + /// The provider JSON produced by `request_from_universal` + provider_json: Value, + /// The universal request recovered by `request_to_universal` + roundtripped: UniversalRequest, +} + +/// Harness for testing roundtrip conversions. Operates on any `UniversalRequest`. +struct RoundtripHarness; + +impl RoundtripHarness { + /// Convert a provider-native JSON payload to Universal first, then roundtrip + /// through the same or different providers. + fn from_provider_json( + source_format: ProviderFormat, + payload: Value, + ) -> Result { + let adapter = adapter_for_format(source_format) + .ok_or_else(|| format!("No adapter for {:?}", source_format))?; + adapter + .request_to_universal(payload) + .map_err(|e| format!("request_to_universal({:?}): {}", source_format, e)) + } + + /// Convert Universal -> Provider -> Universal and return both intermediate + result. + fn self_roundtrip( + req: &UniversalRequest, + format: ProviderFormat, + ) -> Result { + let adapter = + adapter_for_format(format).ok_or_else(|| format!("No adapter for {:?}", format))?; + + let mut req = req.clone(); + if req.model.is_none() { + req.model = Some(default_model(format).to_string()); + } + adapter.apply_defaults(&mut req); + + let provider_json = adapter + .request_from_universal(&req) + .map_err(|e| format!("request_from_universal({:?}): {}", format, e))?; + + let roundtripped = adapter + .request_to_universal(provider_json.clone()) + .map_err(|e| format!("request_to_universal({:?}): {}", format, e))?; + + Ok(RoundtripResult { + provider_json, + roundtripped, + }) + } + + /// Convert Universal -> Source -> Universal -> Target -> Universal. + fn cross_provider( + req: &UniversalRequest, + source: ProviderFormat, + target: ProviderFormat, + ) -> Result { + let source_result = Self::self_roundtrip(req, source)?; + + let target_adapter = + adapter_for_format(target).ok_or_else(|| format!("No adapter for {:?}", target))?; + + let mut universal_for_target = source_result.roundtripped; + universal_for_target.model = Some(default_model(target).to_string()); + target_adapter.apply_defaults(&mut universal_for_target); + + let target_json = target_adapter + .request_from_universal(&universal_for_target) + .map_err(|e| format!("request_from_universal({:?}): {}", target, e))?; + + let roundtripped = target_adapter + .request_to_universal(target_json.clone()) + .map_err(|e| format!("request_to_universal({:?}): {}", target, e))?; + + Ok(RoundtripResult { + provider_json: target_json, + roundtripped, + }) + } +} + +// ============================================================================ +// Content extraction (lossy-conversion-aware comparison) +// ============================================================================ + +/// A normalized per-message summary. Joins text so that `["a","b"]` and `["ab"]` +/// compare equal (providers may merge adjacent text blocks). Ignores `tool_name` +/// on tool results since OpenAI doesn't carry it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct MessageSummary { + role: &'static str, + text: String, + tool_call_ids: Vec, + tool_result_ids: Vec, +} + +fn summarize_messages(messages: &[Message]) -> Vec { + messages + .iter() + .map(|msg| match msg { + Message::System { content } => MessageSummary { + role: "system", + text: join_user_content(content), + tool_call_ids: vec![], + tool_result_ids: vec![], + }, + Message::User { content } => MessageSummary { + role: "user", + text: join_user_content(content), + tool_call_ids: vec![], + tool_result_ids: vec![], + }, + Message::Assistant { content, .. } => { + let (text, tool_call_ids) = summarize_assistant_content(content); + MessageSummary { + role: "assistant", + text, + tool_call_ids, + tool_result_ids: vec![], + } + } + Message::Tool { content } => MessageSummary { + role: "tool", + text: String::new(), + tool_call_ids: vec![], + tool_result_ids: content + .iter() + .map(|p| { + let ToolContentPart::ToolResult(tr) = p; + tr.tool_call_id.clone() + }) + .collect(), + }, + }) + .collect() +} + +fn join_user_content(content: &UserContent) -> String { + match content { + UserContent::String(s) => s.clone(), + UserContent::Array(parts) => parts + .iter() + .filter_map(|p| match p { + UserContentPart::Text(t) => Some(t.text.as_str()), + _ => None, + }) + .collect::>() + .join(""), + } +} + +fn summarize_assistant_content(content: &AssistantContent) -> (String, Vec) { + match content { + AssistantContent::String(s) => (s.clone(), vec![]), + AssistantContent::Array(parts) => { + let mut text = String::new(); + let mut tool_ids = Vec::new(); + for p in parts { + match p { + AssistantContentPart::Text(t) => text.push_str(&t.text), + AssistantContentPart::ToolCall { tool_call_id, .. } => { + tool_ids.push(tool_call_id.clone()); + } + AssistantContentPart::Reasoning { text: r, .. } => { + text.push_str(r); + } + _ => {} + } + } + (text, tool_ids) + } + } +} + +fn non_system(summaries: &[MessageSummary]) -> Vec<&MessageSummary> { + summaries.iter().filter(|s| s.role != "system").collect() +} + +fn extract_system_text(messages: &[Message]) -> String { + let summaries = summarize_messages(messages); + summaries + .iter() + .filter(|s| s.role == "system") + .map(|s| s.text.as_str()) + .collect::>() + .join("\n") +} + +fn default_model(format: ProviderFormat) -> &'static str { + match format { + ProviderFormat::OpenAI => "gpt-4", + ProviderFormat::Anthropic => "claude-3-5-sonnet-20241022", + ProviderFormat::Google => "gemini-1.5-flash", + ProviderFormat::Responses => "gpt-4", + _ => "test-model", + } +} + +fn parse_format(s: &str) -> ProviderFormat { + match s { + "openai" => ProviderFormat::OpenAI, + "anthropic" => ProviderFormat::Anthropic, + "google" => ProviderFormat::Google, + "responses" => ProviderFormat::Responses, + other => panic!("Unknown format in test case: {:?}", other), + } +} + +// ============================================================================ +// Assertion helpers +// ============================================================================ + +fn assert_anthropic_system_preserved(original: &UniversalRequest, anthropic_json: &Value) { + let original_system = extract_system_text(&original.messages); + if original_system.is_empty() { + return; + } + + let system_field = anthropic_json.get("system"); + assert!( + system_field.is_some(), + "Anthropic JSON should have 'system' field.\n\ + Original system text: {:?}\n\ + Anthropic JSON: {}", + original_system, + serde_json::to_string_pretty(anthropic_json).unwrap_or_default() + ); + + let system_text = system_field.unwrap().as_str().unwrap_or(""); + assert!( + !system_text.is_empty(), + "Anthropic 'system' field should not be empty.\n\ + Original system text: {:?}\n\ + Anthropic JSON: {}", + original_system, + serde_json::to_string_pretty(anthropic_json).unwrap_or_default() + ); +} + +fn assert_messages_preserved( + original: &UniversalRequest, + roundtripped: &UniversalRequest, + context: &str, +) { + let orig = summarize_messages(&original.messages); + let rt = summarize_messages(&roundtripped.messages); + + assert_eq!( + non_system(&orig), + non_system(&rt), + "Non-system messages should be preserved in {}.\n\ + Original: {:#?}\n\ + Roundtripped: {:#?}", + context, + non_system(&orig), + non_system(&rt) + ); +} + +// ============================================================================ +// Saved test case loading (roundtrip_cases.json) +// ============================================================================ + +#[derive(serde::Deserialize, Debug)] +struct SavedTestCase { + description: String, + source_format: String, + payload: Value, + /// Provider formats to roundtrip through. Each one runs: + /// source -> Universal -> target -> Universal + roundtrip_through: Vec, +} + +fn load_saved_cases() -> Vec { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/roundtrip_cases.json"); + let content = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("Failed to read {}: {}", path.display(), e)); + serde_json::from_str(&content) + .unwrap_or_else(|e| panic!("Failed to parse {}: {}", path.display(), e)) +} + +/// When proptest finds a failure, call this to get a JSON snippet you can paste +/// into `roundtrip_cases.json`. +fn format_as_saved_case( + description: &str, + source_format: ProviderFormat, + provider_json: &Value, + roundtrip_through: &[&str], +) -> String { + let case = json!({ + "description": description, + "source_format": format!("{}", source_format).to_lowercase(), + "payload": provider_json, + "roundtrip_through": roundtrip_through, + }); + serde_json::to_string_pretty(&case).unwrap() +} + +// ============================================================================ +// Saved case tests +// ============================================================================ + +#[test] +fn saved_cases_self_roundtrip() { + let cases = load_saved_cases(); + for case in &cases { + let source_format = parse_format(&case.source_format); + + // Parse from source format to Universal + let universal = RoundtripHarness::from_provider_json(source_format, case.payload.clone()) + .unwrap_or_else(|e| { + panic!( + "[{}] Failed to parse {} payload: {}", + case.description, case.source_format, e + ) + }); + + // Self-roundtrip through source format + let result = + RoundtripHarness::self_roundtrip(&universal, source_format).unwrap_or_else(|e| { + panic!( + "[{}] Self-roundtrip through {} failed: {}", + case.description, case.source_format, e + ) + }); + + assert_messages_preserved( + &universal, + &result.roundtripped, + &format!( + "[{}] self-roundtrip through {}", + case.description, case.source_format + ), + ); + + // For Anthropic, also check system preservation + if source_format == ProviderFormat::Anthropic { + assert_anthropic_system_preserved(&universal, &result.provider_json); + } + } +} + +#[test] +fn saved_cases_cross_provider() { + let cases = load_saved_cases(); + for case in &cases { + let source_format = parse_format(&case.source_format); + + let universal = RoundtripHarness::from_provider_json(source_format, case.payload.clone()) + .unwrap_or_else(|e| { + panic!( + "[{}] Failed to parse {} payload: {}", + case.description, case.source_format, e + ) + }); + + for target_name in &case.roundtrip_through { + let target_format = parse_format(target_name); + + let result = RoundtripHarness::cross_provider(&universal, source_format, target_format) + .unwrap_or_else(|e| { + panic!( + "[{}] {} -> {} failed: {}", + case.description, case.source_format, target_name, e + ) + }); + + assert_messages_preserved( + &universal, + &result.roundtripped, + &format!( + "[{}] {} -> {}", + case.description, case.source_format, target_name + ), + ); + + // For Anthropic targets, verify system not dropped + if target_format == ProviderFormat::Anthropic { + assert_anthropic_system_preserved(&universal, &result.provider_json); + } + } + } +} + +// ============================================================================ +// Proptest strategies: generate provider-native JSON payloads +// ============================================================================ + +mod strategies { + use super::schema_strategy::{load_openapi_definitions, strategy_for_schema_name}; + use super::*; + use proptest::prelude::*; + + // -- Shared primitives -- + + pub fn arb_text() -> impl Strategy { + "[a-zA-Z0-9 .!?,]{1,80}" + } + + fn arb_json_value() -> impl Strategy { + prop_oneof![ + Just(Value::Null), + any::().prop_map(Value::Bool), + any::().prop_map(|i| json!(i)), + arb_text().prop_map(Value::String), + ] + } + + fn arb_json_object() -> impl Strategy { + proptest::collection::hash_map("[a-z]{2,8}", arb_json_value(), 0..=3) + .prop_map(|m| json!(m.into_iter().collect::>())) + } + + fn arb_function_name() -> impl Strategy { + "[a-z_]{3,12}" + } + + // ======================================================================== + // Schema-driven strategies (OpenAI + Anthropic from OpenAPI specs) + // ======================================================================== + + fn specs_dir() -> String { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + // specs/ is at the repo root, two levels up from crates/lingua/ + format!("{}/../..", manifest_dir) + } + + pub fn arb_openai_payload() -> BoxedStrategy { + let defs = load_openapi_definitions(&format!("{}/specs/openai/openapi.yml", specs_dir())); + strategy_for_schema_name("CreateChatCompletionRequest", &defs) + } + + pub fn arb_anthropic_payload() -> BoxedStrategy { + let defs = + load_openapi_definitions(&format!("{}/specs/anthropic/openapi.yml", specs_dir())); + strategy_for_schema_name("CreateMessageParams", &defs) + } + + // ======================================================================== + // Google GenerateContent JSON strategies (hand-written, no OpenAPI spec) + // ======================================================================== + + fn google_text_part() -> impl Strategy { + arb_text().prop_map(|t| json!({"text": t})) + } + + fn google_function_call_part() -> impl Strategy { + (arb_function_name(), arb_json_object()) + .prop_map(|(name, args)| json!({"functionCall": {"name": name, "args": args}})) + } + + fn google_function_response_part() -> impl Strategy { + (arb_function_name(), arb_json_object()).prop_map( + |(name, response)| json!({"functionResponse": {"name": name, "response": response}}), + ) + } + + fn google_user_content() -> impl Strategy { + proptest::collection::vec(google_text_part(), 1..=3) + .prop_map(|parts| json!({"role": "user", "parts": parts})) + } + + fn google_model_content() -> impl Strategy { + prop_oneof![ + // Text-only + 3 => proptest::collection::vec(google_text_part(), 1..=3) + .prop_map(|parts| json!({"role": "model", "parts": parts})), + // Function calls + 1 => proptest::collection::vec(google_function_call_part(), 1..=2) + .prop_map(|parts| json!({"role": "model", "parts": parts})), + ] + } + + fn google_tool_response_content() -> impl Strategy { + proptest::collection::vec(google_function_response_part(), 1..=2) + .prop_map(|parts| json!({"role": "user", "parts": parts})) + } + + fn google_content() -> impl Strategy { + prop_oneof![ + 3 => google_user_content(), + 3 => google_model_content(), + 1 => google_tool_response_content(), + ] + } + + pub fn arb_google_payload() -> impl Strategy { + ( + proptest::collection::vec(google_content(), 1..=6), + prop::option::of(arb_text()), // optional systemInstruction + ) + .prop_map(|(contents, system)| { + let mut payload = json!({ + "contents": contents, + "generationConfig": {"maxOutputTokens": 1024}, + }); + if let Some(s) = system { + payload["systemInstruction"] = json!({"parts": [{"text": s}]}); + } + payload + }) + } + + // ======================================================================== + // Combined: generate (format, payload) pairs + // ======================================================================== + + pub fn arb_provider_payload() -> impl Strategy { + prop_oneof![ + arb_openai_payload().prop_map(|p| (ProviderFormat::OpenAI, p)), + arb_anthropic_payload().prop_map(|p| (ProviderFormat::Anthropic, p)), + arb_google_payload().prop_map(|p| (ProviderFormat::Google, p)), + ] + } +} + +// ============================================================================ +// Proptest-driven tests +// ============================================================================ + +use proptest::prelude::*; + +/// Shared helper: parse provider JSON to Universal, roundtrip through each target, +/// and check assertions. Returns Err with saveable JSON on failure. +fn run_roundtrips( + source_format: ProviderFormat, + payload: &Value, + targets: &[ProviderFormat], +) -> Result<(), String> { + let universal = RoundtripHarness::from_provider_json(source_format, payload.clone())?; + + // Self-roundtrip + let self_result = RoundtripHarness::self_roundtrip(&universal, source_format)?; + let orig = summarize_messages(&universal.messages); + let rt = summarize_messages(&self_result.roundtripped.messages); + if non_system(&orig) != non_system(&rt) { + return Err(format!( + "Self-roundtrip through {:?} lost messages.\n\ + Original: {:#?}\n\ + Roundtripped: {:#?}\n\ + To save:\n{}", + source_format, + non_system(&orig), + non_system(&rt), + format_as_saved_case( + &format!("proptest: {:?} self-roundtrip failure", source_format), + source_format, + payload, + &[&format!("{}", source_format).to_lowercase()], + ), + )); + } + + // Cross-provider roundtrips + for &target in targets { + if let Ok(result) = RoundtripHarness::cross_provider(&universal, source_format, target) { + let rt = summarize_messages(&result.roundtripped.messages); + if non_system(&orig) != non_system(&rt) { + return Err(format!( + "{:?} -> {:?} lost messages.\n\ + Original: {:#?}\n\ + Roundtripped: {:#?}\n\ + To save:\n{}", + source_format, + target, + non_system(&orig), + non_system(&rt), + format_as_saved_case( + &format!("proptest: {:?}->{:?} failure", source_format, target), + source_format, + payload, + &[ + &format!("{}", source_format).to_lowercase(), + &format!("{}", target).to_lowercase(), + ], + ), + )); + } + + if target == ProviderFormat::Anthropic { + let sys = extract_system_text(&universal.messages); + if !sys.is_empty() { + let s = result.provider_json.get("system"); + if s.is_none() || s.unwrap().as_str().unwrap_or("").is_empty() { + return Err(format!( + "Anthropic system dropped! Original: {:?}\nTo save:\n{}", + sys, + format_as_saved_case( + "proptest: Anthropic system dropped", + source_format, + payload, + &[&format!("{}", source_format).to_lowercase(), "anthropic",], + ), + )); + } + } + } + } + } + + Ok(()) +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: 256, + // Don't write .proptest-regressions files; we use roundtrip_cases.json instead. + failure_persistence: None, + ..ProptestConfig::default() + })] + + /// OpenAI Chat Completions JSON -> roundtrip through all providers. + #[test] + fn prop_openai_payload(payload in strategies::arb_openai_payload()) { + if let Ok(universal) = RoundtripHarness::from_provider_json(ProviderFormat::OpenAI, payload.clone()) { + let _ = universal; // parsed ok + if let Err(e) = run_roundtrips( + ProviderFormat::OpenAI, &payload, + &[ProviderFormat::Anthropic, ProviderFormat::Google], + ) { + prop_assert!(false, "{}", e); + } + } + } + + /// Anthropic Messages JSON -> roundtrip through all providers. + #[test] + fn prop_anthropic_payload(payload in strategies::arb_anthropic_payload()) { + if let Ok(universal) = RoundtripHarness::from_provider_json(ProviderFormat::Anthropic, payload.clone()) { + let _ = universal; + if let Err(e) = run_roundtrips( + ProviderFormat::Anthropic, &payload, + &[ProviderFormat::OpenAI, ProviderFormat::Google], + ) { + prop_assert!(false, "{}", e); + } + } + } + + /// Google GenerateContent JSON -> roundtrip through all providers. + #[test] + fn prop_google_payload(payload in strategies::arb_google_payload()) { + if let Ok(universal) = RoundtripHarness::from_provider_json(ProviderFormat::Google, payload.clone()) { + let _ = universal; + if let Err(e) = run_roundtrips( + ProviderFormat::Google, &payload, + &[ProviderFormat::OpenAI, ProviderFormat::Anthropic], + ) { + prop_assert!(false, "{}", e); + } + } + } + + /// Any provider JSON -> all roundtrips (maximum coverage). + #[test] + fn prop_any_provider((format, payload) in strategies::arb_provider_payload()) { + let all_targets = [ProviderFormat::OpenAI, ProviderFormat::Anthropic, ProviderFormat::Google]; + let targets: Vec<_> = all_targets.iter().copied().filter(|&t| t != format).collect(); + + if let Ok(_) = RoundtripHarness::from_provider_json(format, payload.clone()) { + if let Err(e) = run_roundtrips(format, &payload, &targets) { + prop_assert!(false, "{}", e); + } + } + } +} diff --git a/crates/lingua/tests/roundtrip_cases.json b/crates/lingua/tests/roundtrip_cases.json new file mode 100644 index 0000000..ca23a5c --- /dev/null +++ b/crates/lingua/tests/roundtrip_cases.json @@ -0,0 +1,246 @@ +[ + { + "description": "PR #83 regression: system message with array content (OpenAI format)", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Always be concise."} + ] + }, + { + "role": "user", + "content": "Hello" + } + ] + }, + "roundtrip_through": ["openai", "anthropic", "google"] + }, + { + "description": "PR #83 regression: system message with string content (baseline)", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello" + } + ] + }, + "roundtrip_through": ["openai", "anthropic", "google"] + }, + { + "description": "Multi-turn conversation with tool calls (OpenAI format)", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What is the weather in San Francisco?" + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\":\"San Francisco\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": "{\"temperature\":65,\"unit\":\"F\"}" + } + ] + }, + "roundtrip_through": ["openai", "anthropic"] + }, + { + "description": "Anthropic native format with system + multi-turn", + "source_format": "anthropic", + "payload": { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 1024, + "system": "You are a helpful assistant.", + "messages": [ + { + "role": "user", + "content": "Hello" + }, + { + "role": "assistant", + "content": "Hi there! How can I help?" + }, + { + "role": "user", + "content": "Tell me a joke." + } + ] + }, + "roundtrip_through": ["anthropic", "openai"] + }, + { + "description": "Anthropic native format with array user content", + "source_format": "anthropic", + "payload": { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello, how are you?"} + ] + } + ] + }, + "roundtrip_through": ["anthropic", "openai"] + }, + { + "description": "Google native format with system instruction", + "source_format": "google", + "payload": { + "model": "gemini-1.5-flash", + "systemInstruction": { + "parts": [ + {"text": "You are a helpful assistant."} + ] + }, + "contents": [ + { + "role": "user", + "parts": [ + {"text": "Hello"} + ] + } + ], + "generationConfig": { + "maxOutputTokens": 1024 + } + }, + "roundtrip_through": ["google", "openai", "anthropic"] + }, + { + "description": "OpenAI multi-part user content", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Part one."}, + {"type": "text", "text": "Part two."} + ] + } + ] + }, + "roundtrip_through": ["openai", "anthropic"] + }, + { + "description": "Developer message with array content -> Anthropic system dropped (schema-fuzz)", + "source_format": "openai", + "payload": { + "model": "gpt-4", + "messages": [ + { + "role": "developer", + "content": [ + {"type": "text", "text": "You are a helpful assistant."} + ] + }, + { + "role": "user", + "content": "Hello" + } + ] + }, + "roundtrip_through": ["openai", "anthropic"] + }, + { + "description": "Anthropic thinking block text lost in cross-provider conversion (schema-fuzz)", + "source_format": "anthropic", + "payload": { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 1024, + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "Let me think about this carefully.", + "signature": "sig_abc123" + } + ] + } + ] + }, + "roundtrip_through": ["anthropic", "google"] + }, + { + "description": "Google consecutive same-role messages merged on roundtrip (schema-fuzz)", + "source_format": "google", + "payload": { + "contents": [ + { + "role": "user", + "parts": [{"text": "First message"}] + }, + { + "role": "user", + "parts": [{"text": "Second message"}] + } + ], + "generationConfig": {"maxOutputTokens": 1024} + }, + "roundtrip_through": ["google"] + }, + { + "description": "Google multiple functionResponse parts reduced to one in OpenAI (schema-fuzz)", + "source_format": "google", + "payload": { + "contents": [ + { + "role": "user", + "parts": [{"text": "Call two functions"}] + }, + { + "role": "model", + "parts": [ + {"functionCall": {"name": "get_weather", "args": {"city": "SF"}}}, + {"functionCall": {"name": "get_time", "args": {"city": "SF"}}} + ] + }, + { + "role": "user", + "parts": [ + {"functionResponse": {"name": "get_weather", "response": {"temp": 65}}}, + {"functionResponse": {"name": "get_time", "response": {"time": "3pm"}}} + ] + } + ], + "generationConfig": {"maxOutputTokens": 1024} + }, + "roundtrip_through": ["google", "openai"] + } +] diff --git a/crates/lingua/tests/schema_strategy.rs b/crates/lingua/tests/schema_strategy.rs new file mode 100644 index 0000000..e1f0ff7 --- /dev/null +++ b/crates/lingua/tests/schema_strategy.rs @@ -0,0 +1,335 @@ +//! Generate random `serde_json::Value` from OpenAPI / JSON Schema definitions. +//! +//! This module walks a JSON Schema tree and produces a `proptest::Strategy` that +//! generates conforming JSON values. It handles `$ref`, `const`, `enum`, +//! `anyOf`/`oneOf`, objects (required + optional properties), arrays, and +//! primitive types. +//! +//! Used by `fuzz_roundtrip.rs` to auto-generate provider request payloads +//! directly from the OpenAPI specs in `specs/`. + +use lingua::serde_json::{self, json, Map, Value}; +use proptest::prelude::*; +use std::sync::Arc; + +/// Context for schema resolution (definitions map + depth tracking). +#[derive(Clone)] +pub struct SchemaCtx { + definitions: Arc>, +} + +impl SchemaCtx { + pub fn new(definitions: Map) -> Self { + Self { + definitions: Arc::new(definitions), + } + } + + /// Look up a schema name from a `$ref` string like `#/components/schemas/Foo`. + fn resolve_ref(&self, ref_str: &str) -> Option<&Value> { + let name = ref_str.rsplit('/').next()?; + self.definitions.get(name) + } +} + +const MAX_DEPTH: usize = 12; + +/// Build a proptest `Strategy` that generates random JSON conforming to `schema`. +pub fn strategy_from_schema(schema: &Value, ctx: &SchemaCtx, depth: usize) -> BoxedStrategy { + if depth > MAX_DEPTH { + return Just(Value::Null).boxed(); + } + + // Handle $ref + if let Some(ref_str) = schema.get("$ref").and_then(|v| v.as_str()) { + if let Some(resolved) = ctx.resolve_ref(ref_str) { + let resolved = resolved.clone(); + let ctx = ctx.clone(); + return strategy_from_schema(&resolved, &ctx, depth + 1); + } + return Just(Value::Null).boxed(); + } + + // Handle const + if let Some(const_val) = schema.get("const") { + return Just(const_val.clone()).boxed(); + } + + // Handle enum + if let Some(enum_vals) = schema.get("enum").and_then(|v| v.as_array()) { + if !enum_vals.is_empty() { + let vals: Vec = enum_vals.clone(); + return (0..vals.len()).prop_map(move |i| vals[i].clone()).boxed(); + } + } + + // Handle anyOf / oneOf (pick one variant) + if let Some(variants) = schema + .get("anyOf") + .or_else(|| schema.get("oneOf")) + .and_then(|v| v.as_array()) + { + if !variants.is_empty() { + let strategies: Vec> = variants + .iter() + .map(|v| strategy_from_schema(v, ctx, depth + 1)) + .collect(); + return proptest::strategy::Union::new(strategies).boxed(); + } + } + + // Handle allOf (merge all schemas - simplified: just use the first object-like one) + if let Some(all_of) = schema.get("allOf").and_then(|v| v.as_array()) { + // Merge all properties into a single object schema + let mut merged_props = Map::new(); + let mut merged_required = Vec::new(); + for sub in all_of { + if let Some(props) = sub.get("properties").and_then(|p| p.as_object()) { + for (k, v) in props { + merged_props.insert(k.clone(), v.clone()); + } + } + if let Some(req) = sub.get("required").and_then(|r| r.as_array()) { + for r in req { + if let Some(s) = r.as_str() { + merged_required.push(Value::String(s.to_string())); + } + } + } + // Also resolve $ref within allOf + if let Some(ref_str) = sub.get("$ref").and_then(|v| v.as_str()) { + if let Some(resolved) = ctx.resolve_ref(ref_str) { + if let Some(props) = resolved.get("properties").and_then(|p| p.as_object()) { + for (k, v) in props { + merged_props.insert(k.clone(), v.clone()); + } + } + if let Some(req) = resolved.get("required").and_then(|r| r.as_array()) { + for r in req { + if let Some(s) = r.as_str() { + merged_required.push(Value::String(s.to_string())); + } + } + } + } + } + } + if !merged_props.is_empty() { + let merged = json!({ + "type": "object", + "properties": Value::Object(merged_props), + "required": Value::Array(merged_required), + }); + return strategy_from_schema(&merged, ctx, depth + 1); + } + } + + // Handle by type + let type_str = schema.get("type").and_then(|v| v.as_str()).unwrap_or(""); + + match type_str { + "string" => arb_string(schema).boxed(), + "integer" => (-1000i64..1000).prop_map(|i| json!(i)).boxed(), + "number" => (-100.0f64..100.0).prop_map(|f| json!(f)).boxed(), + "boolean" => any::().prop_map(Value::Bool).boxed(), + "null" => Just(Value::Null).boxed(), + "object" => strategy_for_object(schema, ctx, depth), + "array" => strategy_for_array(schema, ctx, depth), + _ => { + // No type specified - could be a ref-only or untyped schema + // Try to infer from other properties + if schema.get("properties").is_some() { + strategy_for_object(schema, ctx, depth) + } else if schema.get("items").is_some() { + strategy_for_array(schema, ctx, depth) + } else { + // Fallback: random primitive + prop_oneof![ + "[a-zA-Z0-9 ]{1,30}".prop_map(Value::String), + Just(Value::Null), + ] + .boxed() + } + } + } +} + +fn arb_string(schema: &Value) -> impl Strategy { + // If there are enum values, use those + if let Some(enum_vals) = schema.get("enum").and_then(|v| v.as_array()) { + let vals: Vec = enum_vals + .iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect(); + if !vals.is_empty() { + return (0..vals.len()) + .prop_map(move |i| Value::String(vals[i].clone())) + .boxed(); + } + } + "[a-zA-Z0-9 .!?,]{1,50}".prop_map(Value::String).boxed() +} + +fn strategy_for_object(schema: &Value, ctx: &SchemaCtx, depth: usize) -> BoxedStrategy { + let properties = schema + .get("properties") + .and_then(|v| v.as_object()) + .cloned() + .unwrap_or_default(); + + let required: Vec = schema + .get("required") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + + if properties.is_empty() { + return Just(json!({})).boxed(); + } + + // Separate required and optional properties + let mut required_entries: Vec<(String, Value)> = Vec::new(); + let mut optional_entries: Vec<(String, Value)> = Vec::new(); + + for (key, prop_schema) in &properties { + if required.contains(key) { + required_entries.push((key.clone(), prop_schema.clone())); + } else { + optional_entries.push((key.clone(), prop_schema.clone())); + } + } + + let ctx_clone = ctx.clone(); + + // Build strategies for required properties + let required_strategies: Vec<(String, BoxedStrategy)> = required_entries + .into_iter() + .map(|(key, prop_schema)| { + let strat = strategy_from_schema(&prop_schema, &ctx_clone, depth + 1); + (key, strat) + }) + .collect(); + + // Build strategies for optional properties (each may or may not be included) + let optional_strategies: Vec<(String, BoxedStrategy>)> = optional_entries + .into_iter() + .map(|(key, prop_schema)| { + let strat = + prop::option::of(strategy_from_schema(&prop_schema, &ctx_clone, depth + 1)).boxed(); + (key, strat) + }) + .collect(); + + // We need to combine all strategies. Use a tuple approach with vectors. + // Generate required values as a vec, optional as a vec of options. + let req_strat = required_strategies + .into_iter() + .map(|(k, s)| s.prop_map(move |v| (k.clone(), v))) + .collect::>(); + + let opt_strat = optional_strategies + .into_iter() + .map(|(k, s)| s.prop_map(move |v| (k.clone(), v))) + .collect::>(); + + // Use prop_flat_map to combine dynamically-sized strategy lists + let req_values = if req_strat.is_empty() { + Just(Vec::<(String, Value)>::new()).boxed() + } else { + req_strat.into_iter().fold( + Just(Vec::<(String, Value)>::new()).boxed(), + |acc, item_strat| { + (acc, item_strat) + .prop_map(|(mut vec, item)| { + vec.push(item); + vec + }) + .boxed() + }, + ) + }; + + let opt_values = if opt_strat.is_empty() { + Just(Vec::<(String, Option)>::new()).boxed() + } else { + opt_strat.into_iter().fold( + Just(Vec::<(String, Option)>::new()).boxed(), + |acc, item_strat| { + (acc, item_strat) + .prop_map(|(mut vec, item)| { + vec.push(item); + vec + }) + .boxed() + }, + ) + }; + + (req_values, opt_values) + .prop_map(|(required_vals, optional_vals)| { + let mut map = Map::new(); + for (key, val) in required_vals { + map.insert(key, val); + } + for (key, maybe_val) in optional_vals { + if let Some(val) = maybe_val { + map.insert(key, val); + } + } + Value::Object(map) + }) + .boxed() +} + +fn strategy_for_array(schema: &Value, ctx: &SchemaCtx, depth: usize) -> BoxedStrategy { + let items_schema = schema.get("items").cloned().unwrap_or(json!({})); + let min_items = schema.get("minItems").and_then(|v| v.as_u64()).unwrap_or(1) as usize; + let max_items = schema + .get("maxItems") + .and_then(|v| v.as_u64()) + .unwrap_or(3) + .min(5) as usize; + let max_items = max_items.max(min_items); + + let item_strat = strategy_from_schema(&items_schema, ctx, depth + 1); + proptest::collection::vec(item_strat, min_items..=max_items) + .prop_map(Value::Array) + .boxed() +} + +// ============================================================================ +// Spec loading helpers +// ============================================================================ + +/// Load an OpenAPI spec and extract the `components.schemas` definitions map. +pub fn load_openapi_definitions(spec_path: &str) -> Map { + let content = std::fs::read_to_string(spec_path) + .unwrap_or_else(|e| panic!("Failed to read spec at {}: {}", spec_path, e)); + + // Try JSON first (Anthropic spec is JSON despite .yml extension), then YAML + let spec: Value = serde_json::from_str(&content) + .or_else(|_| serde_yaml::from_str::(&content).map_err(|e| e.to_string())) + .unwrap_or_else(|e| panic!("Failed to parse spec at {}: {}", spec_path, e)); + + spec.get("components") + .and_then(|c| c.get("schemas")) + .and_then(|s| s.as_object()) + .cloned() + .unwrap_or_else(|| panic!("No components.schemas in {}", spec_path)) +} + +/// Build a strategy for a named schema from the definitions map. +pub fn strategy_for_schema_name( + name: &str, + definitions: &Map, +) -> BoxedStrategy { + let schema = definitions + .get(name) + .unwrap_or_else(|| panic!("Schema '{}' not found in definitions", name)); + let ctx = SchemaCtx::new(definitions.clone()); + strategy_from_schema(schema, &ctx, 0) +}