feat(persistence): add cache write-back with compression and suppress OpenDAL warnings

CLAUDE.md

@@ -316,9 +316,28 @@ The workspace uses Rust edition 2024 and resolver version 2 for optimal dependen
 - `terraphim_rolegraph`: Knowledge graph implementation with node/edge relationships
 - `terraphim_automata`: Text matching, autocomplete, and thesaurus building
 - `terraphim_config`: Configuration management and role-based settings
-- `terraphim_persistence`: Document storage abstraction layer
+- `terraphim_persistence`: Document storage abstraction layer with cache warm-up
 - `terraphim_server`: HTTP API server (main binary)
 
+### Persistence Layer Cache Warm-up
+
+The persistence layer (`terraphim_persistence`) supports transparent cache warm-up for multi-backend configurations:
+
+**Cache Write-back Behavior:**
+- When data is loaded from a slower fallback operator (e.g., SQLite, S3), it is automatically cached to the fastest operator (e.g., memory, dashmap)
+- Uses fire-and-forget pattern with `tokio::spawn` - non-blocking, doesn't slow load path
+- Objects over 1MB are compressed using zstd before caching
+- Schema evolution handling: if cached data fails to deserialize, the cache entry is deleted and data is refetched from persistent storage
+
+**Configuration:**
+- Operators are ordered by speed (memory > dashmap > sqlite > s3)
+- Same-operator detection: skips redundant cache write if only one backend is configured
+- Tracing spans for observability: `load_from_operator{key}`, `try_read{profile}`, `cache_writeback{key, size}`
+
+**Testing:**
+- Use `DeviceStorage::init_memory_only()` for test isolation (single memory backend)
+- Multi-profile cache write-back tested via integration tests in `tests/persistence_warmup.rs`
+
 **Agent System Crates**:
 - `terraphim_agent_supervisor`: Agent lifecycle management and supervision
 - `terraphim_agent_registry`: Agent discovery and registration

crates/terraphim_agent/tests/integration_tests.rs

@@ -122,12 +122,7 @@ fn parse_config_from_output(output: &str) -> Result<serde_json::Value> {
 
 /// Clean up test files
 fn cleanup_test_files() -> Result<()> {
-    let test_dirs = vec![
-        "/tmp/terraphim_sqlite",
-        "/tmp/dashmaptest",
-        "/tmp/terraphim_rocksdb",
-        "/tmp/opendal",
-    ];
+    let test_dirs = vec!["/tmp/terraphim_sqlite", "/tmp/dashmaptest", "/tmp/opendal"];
 
     for dir in test_dirs {
         if Path::new(dir).exists() {

crates/terraphim_agent/tests/persistence_tests.rs

@@ -48,12 +48,7 @@ fn parse_config_from_output(output: &str) -> Result<serde_json::Value> {
 /// Clean up test persistence files
 fn cleanup_test_persistence() -> Result<()> {
     // Clean up test persistence directories
-    let test_dirs = vec![
-        "/tmp/terraphim_sqlite",
-        "/tmp/dashmaptest",
-        "/tmp/terraphim_rocksdb",
-        "/tmp/opendal",
-    ];
+    let test_dirs = vec!["/tmp/terraphim_sqlite", "/tmp/dashmaptest", "/tmp/opendal"];
 
     for dir in test_dirs {
         if Path::new(dir).exists() {

crates/terraphim_automata/src/lib.rs

@@ -347,8 +347,21 @@ pub async fn load_thesaurus(automata_path: &AutomataPath) -> Result<Thesaurus> {
     }
 
     let contents = match automata_path {
-        AutomataPath::Local(path) => fs::read_to_string(path)?,
-        AutomataPath::Remote(url) => read_url(url.clone()).await?,
+        AutomataPath::Local(path) => {
+            // Check if file exists before attempting to read
+            if !std::path::Path::new(path).exists() {
+                return Err(TerraphimAutomataError::InvalidThesaurus(format!(
+                    "Thesaurus file not found: {}",
+                    path.display()
+                )));
+            }
+            fs::read_to_string(path)?
+        }
+        AutomataPath::Remote(_) => {
+            return Err(TerraphimAutomataError::InvalidThesaurus(
+                "Remote loading is not supported. Enable the 'remote-loading' feature.".to_string(),
+            ));
+        }
     };
 
     let thesaurus = serde_json::from_str(&contents)?;

crates/terraphim_cli/tests/integration_tests.rs

@@ -41,6 +41,15 @@ fn run_cli_json(args: &[&str]) -> Result<serde_json::Value, String> {
         .map_err(|e| format!("Failed to parse JSON: {} - output: {}", e, stdout))
 }
 
+fn assert_no_json_error(json: &serde_json::Value, context: &str) {
+    assert!(
+        json.get("error").is_none(),
+        "{} returned error: {:?}",
+        context,
+        json.get("error")
+    );
+}
+
 #[cfg(test)]
 mod role_switching_tests {
     use super::*;
@@ -143,15 +152,11 @@ mod role_switching_tests {
     #[test]
     #[serial]
     fn test_find_with_explicit_role() {
-        let result = run_cli_json(&["find", "test text", "--role", "Default"]);
+        let result = run_cli_json(&["find", "test text", "--role", "Terraphim Engineer"]);
 
         match result {
             Ok(json) => {
-                // Check if this is an error response or success response
-                if json.get("error").is_some() {
-                    eprintln!("Find with role returned error: {:?}", json);
-                    return;
-                }
+                assert_no_json_error(&json, "Find with role");
                 // Should succeed with the specified role
                 assert!(
                     json.get("text").is_some() || json.get("matches").is_some(),
@@ -167,15 +172,11 @@ mod role_switching_tests {
     #[test]
     #[serial]
     fn test_replace_with_explicit_role() {
-        let result = run_cli_json(&["replace", "test text", "--role", "Default"]);
+        let result = run_cli_json(&["replace", "test text", "--role", "Terraphim Engineer"]);
 
         match result {
             Ok(json) => {
-                // Check if this is an error response
-                if json.get("error").is_some() {
-                    eprintln!("Replace with role returned error: {:?}", json);
-                    return;
-                }
+                assert_no_json_error(&json, "Replace with role");
                 // May have original field or be an error
                 assert!(
                     json.get("original").is_some() || json.get("replaced").is_some(),
@@ -192,15 +193,11 @@ mod role_switching_tests {
     #[test]
     #[serial]
     fn test_thesaurus_with_explicit_role() {
-        let result = run_cli_json(&["thesaurus", "--role", "Default"]);
+        let result = run_cli_json(&["thesaurus", "--role", "Terraphim Engineer"]);
 
         match result {
             Ok(json) => {
-                // Check if this is an error response
-                if json.get("error").is_some() {
-                    eprintln!("Thesaurus with role returned error: {:?}", json);
-                    return;
-                }
+                assert_no_json_error(&json, "Thesaurus with role");
                 // Should have either role or terms field
                 assert!(
                     json.get("role").is_some()
@@ -346,10 +343,18 @@ mod replace_tests {
     #[test]
     #[serial]
     fn test_replace_markdown_format() {
-        let result = run_cli_json(&["replace", "rust programming", "--link-format", "markdown"]);
+        let result = run_cli_json(&[
+            "replace",
+            "rust programming",
+            "--role",
+            "Terraphim Engineer",
+            "--link-format",
+            "markdown",
+        ]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Replace markdown");
                 assert_eq!(json["format"].as_str(), Some("markdown"));
                 assert_eq!(json["original"].as_str(), Some("rust programming"));
                 assert!(json.get("replaced").is_some());
@@ -363,10 +368,18 @@ mod replace_tests {
     #[test]
     #[serial]
     fn test_replace_html_format() {
-        let result = run_cli_json(&["replace", "async tokio", "--link-format", "html"]);
+        let result = run_cli_json(&[
+            "replace",
+            "async tokio",
+            "--role",
+            "Terraphim Engineer",
+            "--link-format",
+            "html",
+        ]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Replace html");
                 assert_eq!(json["format"].as_str(), Some("html"));
             }
             Err(e) => {
@@ -378,10 +391,18 @@ mod replace_tests {
     #[test]
     #[serial]
     fn test_replace_wiki_format() {
-        let result = run_cli_json(&["replace", "docker kubernetes", "--link-format", "wiki"]);
+        let result = run_cli_json(&[
+            "replace",
+            "docker kubernetes",
+            "--role",
+            "Terraphim Engineer",
+            "--link-format",
+            "wiki",
+        ]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Replace wiki");
                 assert_eq!(json["format"].as_str(), Some("wiki"));
             }
             Err(e) => {
@@ -393,10 +414,18 @@ mod replace_tests {
     #[test]
     #[serial]
     fn test_replace_plain_format() {
-        let result = run_cli_json(&["replace", "git github", "--link-format", "plain"]);
+        let result = run_cli_json(&[
+            "replace",
+            "git github",
+            "--role",
+            "Terraphim Engineer",
+            "--link-format",
+            "plain",
+        ]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Replace plain");
                 assert_eq!(json["format"].as_str(), Some("plain"));
                 // Plain format should not modify text
                 assert_eq!(
@@ -414,10 +443,11 @@ mod replace_tests {
     #[test]
     #[serial]
     fn test_replace_default_format_is_markdown() {
-        let result = run_cli_json(&["replace", "test text"]);
+        let result = run_cli_json(&["replace", "test text", "--role", "Terraphim Engineer"]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Replace default format");
                 assert_eq!(
                     json["format"].as_str(),
                     Some("markdown"),
@@ -436,12 +466,15 @@ mod replace_tests {
         let result = run_cli_json(&[
             "replace",
             "some random text without matches xyz123",
+            "--role",
+            "Terraphim Engineer",
             "--format",
             "markdown",
         ]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Replace preserves text");
                 let _original = json["original"].as_str().unwrap();
                 let replaced = json["replaced"].as_str().unwrap();
                 // Text without matches should be preserved
@@ -461,10 +494,11 @@ mod find_tests {
     #[test]
     #[serial]
     fn test_find_basic() {
-        let result = run_cli_json(&["find", "rust async tokio"]);
+        let result = run_cli_json(&["find", "rust async tokio", "--role", "Terraphim Engineer"]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Find basic");
                 assert_eq!(json["text"].as_str(), Some("rust async tokio"));
                 assert!(json.get("matches").is_some());
                 assert!(json.get("count").is_some());
@@ -478,10 +512,11 @@ mod find_tests {
     #[test]
     #[serial]
     fn test_find_returns_array_of_matches() {
-        let result = run_cli_json(&["find", "api server client"]);
+        let result = run_cli_json(&["find", "api server client", "--role", "Terraphim Engineer"]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Find matches array");
                 assert!(json["matches"].is_array(), "Matches should be an array");
             }
             Err(e) => {
@@ -493,10 +528,16 @@ mod find_tests {
     #[test]
     #[serial]
     fn test_find_matches_have_required_fields() {
-        let result = run_cli_json(&["find", "database json config"]);
+        let result = run_cli_json(&[
+            "find",
+            "database json config",
+            "--role",
+            "Terraphim Engineer",
+        ]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Find matches fields");
                 if let Some(matches) = json["matches"].as_array() {
                     for m in matches {
                         assert!(m.get("term").is_some(), "Match should have term");
@@ -516,10 +557,16 @@ mod find_tests {
     #[test]
     #[serial]
     fn test_find_count_matches_array_length() {
-        let result = run_cli_json(&["find", "linux docker kubernetes"]);
+        let result = run_cli_json(&[
+            "find",
+            "linux docker kubernetes",
+            "--role",
+            "Terraphim Engineer",
+        ]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Find count");
                 let count = json["count"].as_u64().unwrap_or(0) as usize;
                 let matches_len = json["matches"].as_array().map(|a| a.len()).unwrap_or(0);
                 assert_eq!(count, matches_len, "Count should match array length");
@@ -538,10 +585,11 @@ mod thesaurus_tests {
     #[test]
     #[serial]
     fn test_thesaurus_basic() {
-        let result = run_cli_json(&["thesaurus"]);
+        let result = run_cli_json(&["thesaurus", "--role", "Terraphim Engineer"]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Thesaurus basic");
                 assert!(json.get("role").is_some());
                 assert!(json.get("name").is_some());
                 assert!(json.get("terms").is_some());
@@ -557,10 +605,11 @@ mod thesaurus_tests {
     #[test]
     #[serial]
     fn test_thesaurus_with_limit() {
-        let result = run_cli_json(&["thesaurus", "--limit", "5"]);
+        let result = run_cli_json(&["thesaurus", "--role", "Terraphim Engineer", "--limit", "5"]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Thesaurus limit");
                 let shown = json["shown_count"].as_u64().unwrap_or(0);
                 assert!(shown <= 5, "Should respect limit");
 
@@ -576,10 +625,11 @@ mod thesaurus_tests {
     #[test]
     #[serial]
     fn test_thesaurus_terms_have_required_fields() {
-        let result = run_cli_json(&["thesaurus", "--limit", "10"]);
+        let result = run_cli_json(&["thesaurus", "--role", "Terraphim Engineer", "--limit", "10"]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Thesaurus terms fields");
                 if let Some(terms) = json["terms"].as_array() {
                     for term in terms {
                         assert!(term.get("id").is_some(), "Term should have id");
@@ -600,10 +650,11 @@ mod thesaurus_tests {
     #[test]
     #[serial]
     fn test_thesaurus_total_count_greater_or_equal_shown() {
-        let result = run_cli_json(&["thesaurus", "--limit", "5"]);
+        let result = run_cli_json(&["thesaurus", "--role", "Terraphim Engineer", "--limit", "5"]);
 
         match result {
             Ok(json) => {
+                assert_no_json_error(&json, "Thesaurus count");
                 let total = json["total_count"].as_u64().unwrap_or(0);
                 let shown = json["shown_count"].as_u64().unwrap_or(0);
                 assert!(total >= shown, "Total count should be >= shown count");

crates/terraphim_persistence/Cargo.toml

@@ -33,6 +33,8 @@ tokio = { version = "1.27", features = ["fs", "macros", "rt-multi-thread", "sync
 regex = "1.11.0"
 rusqlite = { version = "0.32", optional = true }
 chrono = { version = "0.4", features = ["serde"] }
+zstd = "0.13"
+tracing = "0.1"
 
 
 [dev-dependencies]