From 8c93562574077aa1aea8fda06bdb136f0bd43a0e Mon Sep 17 00:00:00 2001 From: Sopaco Date: Thu, 26 Feb 2026 09:52:40 +0800 Subject: [PATCH 1/5] Bump version to 2.1.0 --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae56ffb..7dbc9cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -598,7 +598,7 @@ dependencies = [ [[package]] name = "cortex-mem-cli" -version = "2.0.0" +version = "2.1.0" dependencies = [ "anyhow", "chrono", @@ -627,7 +627,7 @@ dependencies = [ [[package]] name = "cortex-mem-core" -version = "2.0.0" +version = "2.1.0" dependencies = [ "anyhow", "async-trait", @@ -655,7 +655,7 @@ dependencies = [ [[package]] name = "cortex-mem-mcp" -version = "2.0.0" +version = "2.1.0" dependencies = [ "anyhow", "async-trait", @@ -757,7 +757,7 @@ dependencies = [ [[package]] name = "cortex-mem-tools" -version = "2.0.0" +version = "2.1.0" dependencies = [ "anyhow", "async-trait", From 88c881dd29050fa52e4278a68ea5658db1d38685 Mon Sep 17 00:00:00 2001 From: Sopaco Date: Thu, 26 Feb 2026 15:28:48 +0800 Subject: [PATCH 2/5] Add periodic layer generation and indexing Introduce ensure_timeline_layers to create missing L0/L1 files for a timeline. Add generate_layers_every_n_messages config and trigger periodic layer generation when a session reaches the message threshold. Expose MCP RPCs generate_layers and index_memories. Update operations to use SessionManager::add_message (returning full message URI) and to delete associated vectors (L0/L1/L2) before removing the file from storage. --- .../src/automation/layer_generator.rs | 48 ++++++ cortex-mem-core/src/automation/manager.rs | 142 ++++++++++++++++-- cortex-mem-mcp/src/service.rs | 116 +++++++++++++- cortex-mem-tools/src/operations.rs | 57 ++++--- 4 files changed, 328 insertions(+), 35 deletions(-) diff --git a/cortex-mem-core/src/automation/layer_generator.rs b/cortex-mem-core/src/automation/layer_generator.rs index 4f312de..d6ebc46 100644 --- a/cortex-mem-core/src/automation/layer_generator.rs +++ b/cortex-mem-core/src/automation/layer_generator.rs @@ -240,6 +240,54 @@ impl LayerGenerator { Ok(stats) } + /// 🆕 确保特定timeline目录拥有L0/L1层级文件 + /// + /// 用于会话关闭时触发生成，避免频繁更新 + pub async fn ensure_timeline_layers(&self, timeline_uri: &str) -> Result { + info!("开始为timeline生成层级文件: {}", timeline_uri); + + // 扫描timeline下的所有目录 + let mut directories = Vec::new(); + self.scan_recursive(timeline_uri, &mut directories).await?; + + info!("发现 {} 个timeline目录", directories.len()); + + // 检测缺失的 L0/L1 + let missing = self.filter_missing_layers(&directories).await?; + info!("发现 {} 个目录缺失 L0/L1", missing.len()); + + if missing.is_empty() { + return Ok(GenerationStats { + total: 0, + generated: 0, + failed: 0, + }); + } + + let mut stats = GenerationStats { + total: missing.len(), + generated: 0, + failed: 0, + }; + + // 生成层级文件（不需要分批，因为timeline通常不大） + for dir in missing { + match self.generate_layers_for_directory(&dir).await { + Ok(_) => { + stats.generated += 1; + info!("✓ 生成成功: {}", dir); + } + Err(e) => { + stats.failed += 1; + warn!("✗ 生成失败: {} - {}", dir, e); + } + } + } + + info!("Timeline层级生成完成: 成功 {}, 失败 {}", stats.generated, stats.failed); + Ok(stats) + } + /// 为单个目录生成 L0/L1 async fn generate_layers_for_directory(&self, uri: &str) -> Result<()> { debug!("生成层级文件: {}", uri); diff --git a/cortex-mem-core/src/automation/manager.rs b/cortex-mem-core/src/automation/manager.rs index 5dd95f3..c4b0f6b 100644 --- a/cortex-mem-core/src/automation/manager.rs +++ b/cortex-mem-core/src/automation/manager.rs @@ -24,6 +24,8 @@ pub struct AutomationConfig { pub index_batch_delay: u64, /// 🆕 启动时自动生成缺失的 L0/L1 文件 pub auto_generate_layers_on_startup: bool, + /// 🆕 每N条消息触发一次L0/L1生成（0表示禁用） + pub generate_layers_every_n_messages: usize, } impl Default for AutomationConfig { @@ -35,6 +37,7 @@ impl Default for AutomationConfig { index_on_close: true, // 默认会话关闭时索引 index_batch_delay: 2, auto_generate_layers_on_startup: false, // 🆕 默认关闭（避免启动时阻塞） + generate_layers_every_n_messages: 0, // 🆕 默认禁用（避免频繁LLM调用） } } } @@ -100,11 +103,20 @@ impl AutomationManager { let batch_delay = Duration::from_secs(self.config.index_batch_delay); let mut batch_timer: Option = None; + // 🆕 会话消息计数器（用于触发定期L0/L1生成） + let mut session_message_counts: std::collections::HashMap = std::collections::HashMap::new(); + loop { tokio::select! { // 事件处理 Some(event) = event_rx.recv() => { - if let Err(e) = self.handle_event(event, &mut pending_sessions, &mut batch_timer, batch_delay).await { + if let Err(e) = self.handle_event( + event, + &mut pending_sessions, + &mut batch_timer, + batch_delay, + &mut session_message_counts + ).await { warn!("Failed to handle event: {}", e); } } @@ -135,9 +147,64 @@ impl AutomationManager { pending_sessions: &mut HashSet, batch_timer: &mut Option, batch_delay: Duration, + session_message_counts: &mut std::collections::HashMap, ) -> Result<()> { match event { CortexEvent::Session(SessionEvent::MessageAdded { session_id, .. }) => { + // 更新消息计数 + let count = session_message_counts.entry(session_id.clone()).or_insert(0); + *count += 1; + + // 🆕 检查是否需要基于消息数量触发L0/L1生成 + if self.config.generate_layers_every_n_messages > 0 + && *count % self.config.generate_layers_every_n_messages == 0 + { + if let Some(ref generator) = self.layer_generator { + info!( + "Message count threshold reached ({} messages), triggering L0/L1 generation for session: {}", + count, session_id + ); + + // 异步生成L0/L1（避免阻塞） + let generator_clone = generator.clone(); + let indexer_clone = self.indexer.clone(); + let session_id_clone = session_id.clone(); + let auto_index = self.config.auto_index; + + tokio::spawn(async move { + let timeline_uri = format!("cortex://session/{}/timeline", session_id_clone); + + // 生成L0/L1 + match generator_clone.ensure_timeline_layers(&timeline_uri).await { + Ok(stats) => { + info!( + "✓ Periodic L0/L1 generation for {}: total={}, generated={}, failed={}", + session_id_clone, stats.total, stats.generated, stats.failed + ); + + // 生成后索引（如果启用了auto_index） + if auto_index && stats.generated > 0 { + match indexer_clone.index_thread(&session_id_clone).await { + Ok(index_stats) => { + info!( + "✓ L0/L1 indexed for {}: {} indexed", + session_id_clone, index_stats.total_indexed + ); + } + Err(e) => { + warn!("✗ Failed to index L0/L1 for {}: {}", session_id_clone, e); + } + } + } + } + Err(e) => { + warn!("✗ Periodic L0/L1 generation failed for {}: {}", session_id_clone, e); + } + } + }); + } + } + if self.config.index_on_message { // 实时索引模式：立即索引 info!("Real-time indexing session: {}", session_id); @@ -155,26 +222,75 @@ impl AutomationManager { CortexEvent::Session(SessionEvent::Closed { session_id }) => { if self.config.index_on_close { - info!("Session closed, triggering full processing: {}", session_id); + info!("Session closed, triggering async full processing: {}", session_id); + + // 🔧 异步执行所有后处理任务，避免阻塞事件循环 + let extractor = self.extractor.clone(); + let generator = self.layer_generator.clone(); + let indexer = self.indexer.clone(); + let auto_extract = self.config.auto_extract; + let auto_index = self.config.auto_index; + let session_id_clone = session_id.clone(); - // 1. 自动提取记忆（如果配置了且有extractor） - if self.config.auto_extract { - if let Some(ref extractor) = self.extractor { - match extractor.extract_session(&session_id).await { + tokio::spawn(async move { + let start = tokio::time::Instant::now(); + + // 1. 自动提取记忆（如果配置了且有extractor） + if auto_extract { + if let Some(ref extractor) = extractor { + match extractor.extract_session(&session_id_clone).await { + Ok(stats) => { + info!("✓ Extraction completed for {}: {:?}", session_id_clone, stats); + } + Err(e) => { + warn!("✗ Extraction failed for {}: {}", session_id_clone, e); + } + } + } + } + + // 2. 生成 L0/L1 层级文件（如果配置了layer_generator） + if let Some(ref generator) = generator { + info!("Generating L0/L1 layers for session: {}", session_id_clone); + let timeline_uri = format!("cortex://session/{}/timeline", session_id_clone); + + match generator.ensure_timeline_layers(&timeline_uri).await { Ok(stats) => { - info!("Extraction completed for {}: {:?}", session_id, stats); + info!( + "✓ L0/L1 generation completed for {}: total={}, generated={}, failed={}", + session_id_clone, stats.total, stats.generated, stats.failed + ); } Err(e) => { - warn!("Extraction failed for {}: {}", session_id, e); + warn!("✗ L0/L1 generation failed for {}: {}", session_id_clone, e); } } } - } + + // 3. 索引整个会话（包括新生成的L0/L1/L2） + if auto_index { + match indexer.index_thread(&session_id_clone).await { + Ok(stats) => { + info!( + "✓ Session {} indexed: {} indexed, {} skipped, {} errors", + session_id_clone, stats.total_indexed, stats.total_skipped, stats.total_errors + ); + } + Err(e) => { + warn!("✗ Failed to index session {}: {}", session_id_clone, e); + } + } + } + + let duration = start.elapsed(); + info!( + "🎉 Session {} post-processing completed in {:.2}s", + session_id_clone, + duration.as_secs_f64() + ); + }); - // 2. 索引整个会话（包括L0/L1/L2） - if self.config.auto_index { - self.index_session(&session_id).await?; - } + info!("Session {} close acknowledged, post-processing running in background", session_id); } } diff --git a/cortex-mem-mcp/src/service.rs b/cortex-mem-mcp/src/service.rs index acb74b6..9d7bfc9 100644 --- a/cortex-mem-mcp/src/service.rs +++ b/cortex-mem-mcp/src/service.rs @@ -127,6 +127,37 @@ pub struct GetAbstractResult { abstract_text: String, } +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct GenerateLayersArgs { + /// Thread/session ID (optional, if not provided, generates for all sessions) + thread_id: Option, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct GenerateLayersResult { + success: bool, + message: String, + total: usize, + generated: usize, + failed: usize, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct IndexMemoriesArgs { + /// Thread/session ID (optional, if not provided, indexes all files) + thread_id: Option, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct IndexMemoriesResult { + success: bool, + message: String, + total_files: usize, + indexed_files: usize, + skipped_files: usize, + error_files: usize, +} + // ==================== MCP Tools Implementation ==================== #[tool_router] @@ -149,13 +180,20 @@ impl MemoryMcpService { let role = params.0.role.as_deref().unwrap_or("user"); match self.operations.add_message(&thread_id, role, ¶ms.0.content).await { - Ok(message_id) => { - let uri = format!("cortex://session/{}/timeline/{}.md", thread_id, message_id); - info!("Memory stored at: {}", uri); + Ok(message_uri) => { + // Extract message_id from URI (last segment without extension) + let message_id = message_uri + .rsplit('/') + .next() + .and_then(|s| s.strip_suffix(".md")) + .unwrap_or("unknown") + .to_string(); + + info!("Memory stored at: {}", message_uri); Ok(Json(StoreMemoryResult { success: true, - uri, + uri: message_uri, message_id, })) } @@ -350,6 +388,74 @@ impl MemoryMcpService { } } } + + #[tool(description = "Generate L0/L1 layer files for memories")] + async fn generate_layers( + &self, + params: Parameters, + ) -> std::result::Result, String> { + debug!("generate_layers called with args: {:?}", params.0); + + match self.operations.ensure_all_layers().await { + Ok(stats) => { + let message = if let Some(ref thread_id) = params.0.thread_id { + format!("Generated layers for session {}", thread_id) + } else { + "Generated layers for all sessions".to_string() + }; + + info!("{}: total={}, generated={}, failed={}", + message, stats.total, stats.generated, stats.failed); + + Ok(Json(GenerateLayersResult { + success: true, + message, + total: stats.total, + generated: stats.generated, + failed: stats.failed, + })) + } + Err(e) => { + error!("Failed to generate layers: {}", e); + Err(format!("Failed to generate layers: {}", e)) + } + } + } + + #[tool(description = "Index memories to vector database")] + async fn index_memories( + &self, + params: Parameters, + ) -> std::result::Result, String> { + debug!("index_memories called with args: {:?}", params.0); + + match self.operations.index_all_files().await { + Ok(stats) => { + let message = if let Some(ref thread_id) = params.0.thread_id { + format!("Indexed memories for session {}", thread_id) + } else { + "Indexed all memory files".to_string() + }; + + info!("{}: total={}, indexed={}, skipped={}, errors={}", + message, stats.total_files, stats.indexed_files, + stats.skipped_files, stats.error_files); + + Ok(Json(IndexMemoriesResult { + success: true, + message, + total_files: stats.total_files, + indexed_files: stats.indexed_files, + skipped_files: stats.skipped_files, + error_files: stats.error_files, + })) + } + Err(e) => { + error!("Failed to index memories: {}", e); + Err(format!("Failed to index memories: {}", e)) + } + } + } } #[tool_handler] @@ -366,6 +472,8 @@ impl ServerHandler for MemoryMcpService { - get_memory: Retrieve a specific memory\n\ - delete_memory: Delete a memory\n\ - get_abstract: Get the abstract summary of a memory\n\ + - generate_layers: Generate L0/L1 layer files for memories\n\ + - index_memories: Index memories to vector database\n\ \n\ URI format: cortex://{dimension}/{category}/{resource}\n\ Examples:\n\ diff --git a/cortex-mem-tools/src/operations.rs b/cortex-mem-tools/src/operations.rs index ee07ae8..e734ff4 100644 --- a/cortex-mem-tools/src/operations.rs +++ b/cortex-mem-tools/src/operations.rs @@ -13,7 +13,7 @@ use cortex_mem_core::{ LayerGenerator, LayerGenerationConfig, AbstractConfig, OverviewConfig, // 🆕 添加LayerGenerator }, embedding::{EmbeddingClient, EmbeddingConfig}, - vector_store::QdrantVectorStore, + vector_store::{QdrantVectorStore, VectorStore}, // 🔧 添加VectorStore trait events::EventBus, // 🆕 添加EventBus }; use std::sync::Arc; @@ -174,10 +174,11 @@ impl MemoryOperations { let automation_config = AutomationConfig { auto_index: true, auto_extract: false, // Extract由单独的监听器处理 - index_on_message: true, // ✅ 消息时自动索引 - index_on_close: false, // Session关闭时不索引（已经实时索引了） + index_on_message: true, // ✅ 消息时自动索引L2 + index_on_close: true, // ✅ Session关闭时生成L0/L1并索引 index_batch_delay: 1, auto_generate_layers_on_startup: false, // 🆕 启动时不生成（避免阻塞） + generate_layers_every_n_messages: 5, // 🆕 每5条消息生成一次L0/L1 }; // 🆕 创建LayerGenerator（用于退出时手动生成） @@ -353,22 +354,27 @@ impl MemoryOperations { let sm = self.session_manager.read().await; - let message = cortex_mem_core::Message::new( - match role { - "user" => cortex_mem_core::MessageRole::User, - "assistant" => cortex_mem_core::MessageRole::Assistant, - "system" => cortex_mem_core::MessageRole::System, - _ => cortex_mem_core::MessageRole::User, - }, - content, + // 🔧 使用SessionManager::add_message()替代message_storage().save_message() + // 这样可以自动触发MessageAdded事件，从而触发自动索引 + let message_role = match role { + "user" => cortex_mem_core::MessageRole::User, + "assistant" => cortex_mem_core::MessageRole::Assistant, + "system" => cortex_mem_core::MessageRole::System, + _ => cortex_mem_core::MessageRole::User, + }; + + let message = sm.add_message(thread_id, message_role, content.to_string()).await?; + let message_uri = format!( + "cortex://session/{}/timeline/{}/{}/{}_{}.md", + thread_id, + message.timestamp.format("%Y-%m"), + message.timestamp.format("%d"), + message.timestamp.format("%H_%M_%S"), + &message.id[..8] ); - let message_uri = sm.message_storage().save_message(thread_id, &message).await?; - - let message_id = message_uri.rsplit('/').next().unwrap_or("unknown").to_string(); - - tracing::info!("Added message {} to session {}", message_id, thread_id); - Ok(message_id) + tracing::info!("Added message to session {}, URI: {}", thread_id, message_uri); + Ok(message_uri) } /// List sessions @@ -443,8 +449,23 @@ impl MemoryOperations { /// Delete file or directory pub async fn delete(&self, uri: &str) -> Result<()> { + // First delete from vector database + // We need to delete all 3 layers: L0, L1, L2 + let l0_id = cortex_mem_core::uri_to_vector_id(uri, cortex_mem_core::ContextLayer::L0Abstract); + let l1_id = cortex_mem_core::uri_to_vector_id(uri, cortex_mem_core::ContextLayer::L1Overview); + let l2_id = cortex_mem_core::uri_to_vector_id(uri, cortex_mem_core::ContextLayer::L2Detail); + + // Delete from vector store (ignore errors as vectors might not exist) + let _ = self.vector_store.delete(&l0_id).await; + let _ = self.vector_store.delete(&l1_id).await; + let _ = self.vector_store.delete(&l2_id).await; + + tracing::info!("Deleted vectors for URI: {} (L0: {}, L1: {}, L2: {})", + uri, l0_id, l1_id, l2_id); + + // Then delete from filesystem self.filesystem.delete(uri).await?; - tracing::info!("Deleted: {}", uri); + tracing::info!("Deleted file: {}", uri); Ok(()) } From 17b2f88d4e532c120f2e151d3ff35b135e3da159 Mon Sep 17 00:00:00 2001 From: Sopaco Date: Thu, 26 Feb 2026 19:29:30 +0800 Subject: [PATCH 3/5] Add layers, scoped search, and sync_specific_path --- cortex-mem-cli/src/commands/mod.rs | 8 +- cortex-mem-core/Cargo.toml | 2 +- cortex-mem-core/src/automation/indexer.rs | 116 ++++--- .../src/automation/layer_generator.rs | 210 ++++++------ cortex-mem-core/src/automation/manager.rs | 159 +++++---- cortex-mem-core/src/automation/mod.rs | 11 +- cortex-mem-core/src/automation/sync.rs | 39 +++ cortex-mem-core/src/builder.rs | 2 +- cortex-mem-core/src/config.rs | 10 +- cortex-mem-core/src/embedding/mod.rs | 6 +- cortex-mem-core/src/filesystem/operations.rs | 98 +++--- cortex-mem-core/src/layers/mod.rs | 2 +- cortex-mem-core/src/lib.rs | 23 +- cortex-mem-core/src/search/vector_engine.rs | 45 ++- cortex-mem-core/src/session/extraction.rs | 36 +- cortex-mem-core/src/session/manager.rs | 236 +++++++------ cortex-mem-core/src/vector_store/qdrant.rs | 145 ++++---- cortex-mem-insights/src/lib/api.ts | 2 +- cortex-mem-mcp/src/service.rs | 164 ++++++--- cortex-mem-rig/src/lib.rs | 39 +-- cortex-mem-service/src/handlers/automation.rs | 49 +-- cortex-mem-service/src/state.rs | 39 +-- cortex-mem-tools/src/lib.rs | 10 +- cortex-mem-tools/src/operations.rs | 319 ++++++++++++------ cortex-mem-tools/src/tools/storage.rs | 102 ++++-- examples/cortex-mem-tars/src/agent.rs | 117 ++++--- examples/cortex-mem-tars/src/app.rs | 16 +- .../src/audio_transcription.rs | 4 +- 28 files changed, 1227 insertions(+), 782 deletions(-) diff --git a/cortex-mem-cli/src/commands/mod.rs b/cortex-mem-cli/src/commands/mod.rs index 5ccf977..7e66fea 100644 --- a/cortex-mem-cli/src/commands/mod.rs +++ b/cortex-mem-cli/src/commands/mod.rs @@ -1,8 +1,8 @@ pub mod add; -pub mod search; -pub mod list; -pub mod get; pub mod delete; +pub mod get; +pub mod layers; +pub mod list; +pub mod search; pub mod session; pub mod stats; -pub mod layers; // 🆕 层级文件管理 \ No newline at end of file diff --git a/cortex-mem-core/Cargo.toml b/cortex-mem-core/Cargo.toml index e6503f6..8258fda 100644 --- a/cortex-mem-core/Cargo.toml +++ b/cortex-mem-core/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true [dependencies] # Workspace dependencies tokio = { workspace = true } -futures = { workspace = true } # 🆕 用于并发操作 +futures = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } anyhow = { workspace = true } diff --git a/cortex-mem-core/src/automation/indexer.rs b/cortex-mem-core/src/automation/indexer.rs index 6770049..a0ff8db 100644 --- a/cortex-mem-core/src/automation/indexer.rs +++ b/cortex-mem-core/src/automation/indexer.rs @@ -1,10 +1,9 @@ use crate::{ + ContextLayer, Result, embedding::EmbeddingClient, filesystem::{CortexFilesystem, FilesystemOperations}, session::Message, vector_store::{QdrantVectorStore, VectorStore}, - ContextLayer, - Result, }; use std::sync::Arc; use tracing::{debug, info, warn}; @@ -226,12 +225,14 @@ impl AutoIndexer { stats.total_indexed, stats.total_skipped, stats.total_errors ); - // 🆕 Phase 1: Index L0/L1 layers for timeline directories + // Index L0/L1 layers for timeline directories info!("Indexing timeline L0/L1 layers for thread: {}", thread_id); match self.index_timeline_layers(thread_id).await { Ok(layer_stats) => { - info!("Timeline layers indexed: {} L0, {} L1", - layer_stats.l0_indexed, layer_stats.l1_indexed); + info!( + "Timeline layers indexed: {} L0, {} L1", + layer_stats.l0_indexed, layer_stats.l1_indexed + ); stats.total_indexed += layer_stats.l0_indexed + layer_stats.l1_indexed; stats.total_errors += layer_stats.errors; } @@ -295,7 +296,7 @@ impl AutoIndexer { .await?; } else if entry.name.ends_with(".md") && !entry.name.starts_with('.') { if let Ok(content) = self.filesystem.as_ref().read(&entry.uri).await { - // 🆕 先尝试解析为标准markdown格式 + // 先尝试解析为标准markdown格式 if let Some(message) = self.parse_message_markdown(&content) { messages.push(message); } else { @@ -303,9 +304,11 @@ impl AutoIndexer { // 文件名格式：HH_MM_SS_.md // 例如：15_10_18_28b538d8.md // 但这只是UUID的前8字符，我们需要从文件内容中提取完整UUID - + // 尝试从Markdown内容中手动提取ID（更宽松的解析） - let message_id = if let Some(id) = Self::extract_id_from_content(&content) { + let message_id = if let Some(id) = + Self::extract_id_from_content(&content) + { id } else { // 如果仍然提取不到，尝试从文件名提取UUID部分 @@ -316,7 +319,10 @@ impl AutoIndexer { // 取最后一个部分（UUID前8字符） // 但我们知道这不是完整UUID，所以给它一个警告 let partial_id = parts[parts.len() - 1]; - warn!("Could not extract full UUID from {}, using partial ID: {}", entry.uri, partial_id); + warn!( + "Could not extract full UUID from {}, using partial ID: {}", + entry.uri, partial_id + ); // 跳过这个消息，因为部分ID无法用于向量存储 continue; } else { @@ -324,20 +330,23 @@ impl AutoIndexer { continue; } }; - + // 从entry.modified获取时间戳 let timestamp = entry.modified; - + let message = Message { - id: message_id.clone(), // 🔧 clone以便后续使用 + id: message_id.clone(), // 🔧 clone以便后续使用 role: crate::session::MessageRole::User, // 默认为User content: content.trim().to_string(), timestamp, created_at: timestamp, metadata: None, }; - - debug!("Collected message from {} with ID: {}", entry.uri, message_id); + + debug!( + "Collected message from {} with ID: {}", + entry.uri, message_id + ); messages.push(message); } } @@ -372,7 +381,11 @@ impl AutoIndexer { .map(|s| s.trim()) .and_then(|s| { // 移除可能的`符号 - s.trim_start_matches('`').trim_end_matches('`').trim().to_string().into() + s.trim_start_matches('`') + .trim_end_matches('`') + .trim() + .to_string() + .into() }) { if !id_str.is_empty() { @@ -425,12 +438,8 @@ impl AutoIndexer { if line.contains("**ID**:") || line.contains("ID:") { // 尝试提取ID if let Some(id_part) = line.split(':').nth(1) { - let id = id_part - .trim() - .trim_matches('`') - .trim() - .to_string(); - + let id = id_part.trim().trim_matches('`').trim().to_string(); + // 验证是否是有效的UUID格式 if uuid::Uuid::parse_str(&id).is_ok() { return Some(id); @@ -451,23 +460,26 @@ impl AutoIndexer { format!("{:x}", hasher.finish()) } - /// 🆕 索引timeline目录的L0/L1层 - /// + /// 索引timeline目录的L0/L1层 + /// /// 该方法会递归扫描timeline目录结构，为每个包含.abstract.md和.overview.md的目录 /// 生成L0/L1层的向量索引 async fn index_timeline_layers(&self, thread_id: &str) -> Result { let mut stats = TimelineLayerStats::default(); let timeline_base = format!("cortex://session/{}/timeline", thread_id); - + // 递归收集所有timeline目录 let directories = self.collect_timeline_directories(&timeline_base).await?; info!("Found {} timeline directories to index", directories.len()); - + for dir_uri in directories { // 索引L0 Abstract let l0_file_uri = format!("{}/.abstract.md", dir_uri); if let Ok(l0_content) = self.filesystem.as_ref().read(&l0_file_uri).await { - match self.index_layer(&dir_uri, &l0_content, ContextLayer::L0Abstract).await { + match self + .index_layer(&dir_uri, &l0_content, ContextLayer::L0Abstract) + .await + { Ok(indexed) => { if indexed { stats.l0_indexed += 1; @@ -480,11 +492,14 @@ impl AutoIndexer { } } } - + // 索引L1 Overview let l1_file_uri = format!("{}/.overview.md", dir_uri); if let Ok(l1_content) = self.filesystem.as_ref().read(&l1_file_uri).await { - match self.index_layer(&dir_uri, &l1_content, ContextLayer::L1Overview).await { + match self + .index_layer(&dir_uri, &l1_content, ContextLayer::L1Overview) + .await + { Ok(indexed) => { if indexed { stats.l1_indexed += 1; @@ -498,17 +513,18 @@ impl AutoIndexer { } } } - + Ok(stats) } - + /// 收集timeline目录结构中的所有目录URI async fn collect_timeline_directories(&self, base_uri: &str) -> Result> { let mut directories = Vec::new(); - self.collect_directories_recursive(base_uri, &mut directories).await?; + self.collect_directories_recursive(base_uri, &mut directories) + .await?; Ok(directories) } - + /// 递归收集目录 fn collect_directories_recursive<'a>( &'a self, @@ -519,18 +535,19 @@ impl AutoIndexer { match self.filesystem.as_ref().list(uri).await { Ok(entries) => { // 检查当前目录是否包含.abstract.md或.overview.md - let has_layers = entries.iter().any(|e| { - e.name == ".abstract.md" || e.name == ".overview.md" - }); - + let has_layers = entries + .iter() + .any(|e| e.name == ".abstract.md" || e.name == ".overview.md"); + if has_layers { directories.push(uri.to_string()); } - + // 递归处理子目录 for entry in entries { if entry.is_directory && !entry.name.starts_with('.') { - self.collect_directories_recursive(&entry.uri, directories).await?; + self.collect_directories_recursive(&entry.uri, directories) + .await?; } } Ok(()) @@ -542,30 +559,25 @@ impl AutoIndexer { } }) } - + /// 索引单个层（L0或L1） - /// + /// /// 返回: Ok(true)表示已索引, Ok(false)表示已存在跳过 - async fn index_layer( - &self, - dir_uri: &str, - content: &str, - layer: ContextLayer, - ) -> Result { - use crate::vector_store::{uri_to_vector_id, VectorStore}; - + async fn index_layer(&self, dir_uri: &str, content: &str, layer: ContextLayer) -> Result { + use crate::vector_store::{VectorStore, uri_to_vector_id}; + // 生成向量ID（基于目录URI，不是文件URI） let vector_id = uri_to_vector_id(dir_uri, layer); - + // 检查是否已索引 if let Ok(Some(_)) = self.vector_store.as_ref().get(&vector_id).await { debug!("Layer {:?} already indexed for {}", layer, dir_uri); return Ok(false); } - + // 生成embedding let embedding = self.embedding.embed(content).await?; - + // 创建Memory对象 let memory = crate::types::Memory { id: vector_id, @@ -588,7 +600,7 @@ impl AutoIndexer { custom: std::collections::HashMap::new(), }, }; - + // 存储到Qdrant self.vector_store.as_ref().insert(&memory).await?; Ok(true) diff --git a/cortex-mem-core/src/automation/layer_generator.rs b/cortex-mem-core/src/automation/layer_generator.rs index d6ebc46..ba6a941 100644 --- a/cortex-mem-core/src/automation/layer_generator.rs +++ b/cortex-mem-core/src/automation/layer_generator.rs @@ -1,10 +1,10 @@ -use crate::{CortexFilesystem, FilesystemOperations, Result}; -use crate::llm::LLMClient; use crate::layers::generator::{AbstractGenerator, OverviewGenerator}; -use std::sync::Arc; -use tracing::{info, warn, debug}; +use crate::llm::LLMClient; +use crate::{CortexFilesystem, FilesystemOperations, Result}; +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use chrono::{Utc, DateTime}; +use std::sync::Arc; +use tracing::{debug, info, warn}; /// 层级生成配置 #[derive(Debug, Clone)] @@ -67,7 +67,7 @@ pub struct GenerationStats { } /// 层级生成器 -/// +/// /// 负责扫描文件系统，检测缺失的 L0/L1 文件，并渐进式生成 pub struct LayerGenerator { filesystem: Arc, @@ -91,15 +91,15 @@ impl LayerGenerator { config, } } - + /// 扫描所有目录 pub async fn scan_all_directories(&self) -> Result> { let mut directories = Vec::new(); - + // 扫描四个核心维度 for scope in &["session", "user", "agent", "resources"] { let scope_uri = format!("cortex://{}", scope); - + // 检查维度是否存在 if self.filesystem.exists(&scope_uri).await? { match self.scan_scope(&scope_uri).await { @@ -110,17 +110,17 @@ impl LayerGenerator { } } } - + Ok(directories) } - + /// 扫描单个维度 async fn scan_scope(&self, scope_uri: &str) -> Result> { let mut directories = Vec::new(); self.scan_recursive(scope_uri, &mut directories).await?; Ok(directories) } - + /// 递归扫描目录 fn scan_recursive<'a>( &'a self, @@ -136,41 +136,41 @@ impl LayerGenerator { return Ok(()); } }; - + for entry in entries { // 跳过隐藏文件 if entry.name.starts_with('.') { continue; } - + if entry.is_directory { // 添加目录到列表 directories.push(entry.uri.clone()); - + // 递归扫描子目录 self.scan_recursive(&entry.uri, directories).await?; } } - + Ok(()) }) } - + /// 检测目录是否有 L0/L1 文件 pub async fn has_layers(&self, uri: &str) -> Result { let abstract_path = format!("{}/.abstract.md", uri); let overview_path = format!("{}/.overview.md", uri); - + let has_abstract = self.filesystem.exists(&abstract_path).await?; let has_overview = self.filesystem.exists(&overview_path).await?; - + Ok(has_abstract && has_overview) } - + /// 过滤出缺失 L0/L1 的目录 pub async fn filter_missing_layers(&self, dirs: &[String]) -> Result> { let mut missing = Vec::new(); - + for dir in dirs { match self.has_layers(dir).await { Ok(has) => { @@ -183,20 +183,20 @@ impl LayerGenerator { } } } - + Ok(missing) } - + /// 确保所有目录拥有 L0/L1 pub async fn ensure_all_layers(&self) -> Result { info!("开始扫描目录..."); let directories = self.scan_all_directories().await?; info!("发现 {} 个目录", directories.len()); - + info!("检测缺失的 L0/L1..."); let missing = self.filter_missing_layers(&directories).await?; info!("发现 {} 个目录缺失 L0/L1", missing.len()); - + if missing.is_empty() { return Ok(GenerationStats { total: 0, @@ -204,19 +204,19 @@ impl LayerGenerator { failed: 0, }); } - + let mut stats = GenerationStats { total: missing.len(), generated: 0, failed: 0, }; - + // 分批生成 let total_batches = (missing.len() + self.config.batch_size - 1) / self.config.batch_size; - + for (batch_idx, batch) in missing.chunks(self.config.batch_size).enumerate() { info!("处理批次 {}/{}", batch_idx + 1, total_batches); - + for dir in batch { match self.generate_layers_for_directory(dir).await { Ok(_) => { @@ -229,33 +229,32 @@ impl LayerGenerator { } } } - + // 批次间延迟 if batch_idx < total_batches - 1 { tokio::time::sleep(tokio::time::Duration::from_millis(self.config.delay_ms)).await; } } - + info!("生成完成: 成功 {}, 失败 {}", stats.generated, stats.failed); Ok(stats) } - - /// 🆕 确保特定timeline目录拥有L0/L1层级文件 - /// + + /// 确保特定timeline目录拥有L0/L1层级文件 /// 用于会话关闭时触发生成，避免频繁更新 pub async fn ensure_timeline_layers(&self, timeline_uri: &str) -> Result { info!("开始为timeline生成层级文件: {}", timeline_uri); - + // 扫描timeline下的所有目录 let mut directories = Vec::new(); self.scan_recursive(timeline_uri, &mut directories).await?; - + info!("发现 {} 个timeline目录", directories.len()); - + // 检测缺失的 L0/L1 let missing = self.filter_missing_layers(&directories).await?; info!("发现 {} 个目录缺失 L0/L1", missing.len()); - + if missing.is_empty() { return Ok(GenerationStats { total: 0, @@ -263,13 +262,13 @@ impl LayerGenerator { failed: 0, }); } - + let mut stats = GenerationStats { total: missing.len(), generated: 0, failed: 0, }; - + // 生成层级文件（不需要分批，因为timeline通常不大） for dir in missing { match self.generate_layers_for_directory(&dir).await { @@ -283,57 +282,70 @@ impl LayerGenerator { } } } - - info!("Timeline层级生成完成: 成功 {}, 失败 {}", stats.generated, stats.failed); + + info!( + "Timeline层级生成完成: 成功 {}, 失败 {}", + stats.generated, stats.failed + ); Ok(stats) } - + /// 为单个目录生成 L0/L1 async fn generate_layers_for_directory(&self, uri: &str) -> Result<()> { debug!("生成层级文件: {}", uri); - - // 🆕 1. 检查是否需要重新生成（避免重复生成未变更的内容） + + // 1. 检查是否需要重新生成（避免重复生成未变更的内容） if !self.should_regenerate(uri).await? { debug!("目录内容未变更，跳过生成: {}", uri); return Ok(()); } - + // 2. 读取目录内容（聚合所有子文件） let content = self.aggregate_directory_content(uri).await?; - + if content.is_empty() { debug!("目录为空，跳过: {}", uri); return Ok(()); } - + // 3. 使用现有的 AbstractGenerator 生成 L0 抽象 - let abstract_text = self.abstract_gen.generate_with_llm(&content, &self.llm_client).await?; - + let abstract_text = self + .abstract_gen + .generate_with_llm(&content, &self.llm_client) + .await?; + // 4. 使用现有的 OverviewGenerator 生成 L1 概览 - let overview = self.overview_gen.generate_with_llm(&content, &self.llm_client).await?; - + let overview = self + .overview_gen + .generate_with_llm(&content, &self.llm_client) + .await?; + // 5. 强制执行长度限制 let abstract_text = self.enforce_abstract_limit(abstract_text)?; let overview = self.enforce_overview_limit(overview)?; - + // 6. 添加 "Added" 日期标记（与 extraction.rs 保持一致） let timestamp = Utc::now().format("%Y-%m-%d %H:%M:%S UTC"); let abstract_with_date = format!("{}\n\n**Added**: {}", abstract_text, timestamp); let overview_with_date = format!("{}\n\n---\n\n**Added**: {}", overview, timestamp); - + // 7. 写入文件 let abstract_path = format!("{}/.abstract.md", uri); let overview_path = format!("{}/.overview.md", uri); - - self.filesystem.write(&abstract_path, &abstract_with_date).await?; - self.filesystem.write(&overview_path, &overview_with_date).await?; - + + self.filesystem + .write(&abstract_path, &abstract_with_date) + .await?; + self.filesystem + .write(&overview_path, &overview_with_date) + .await?; + debug!("层级文件生成完成: {}", uri); Ok(()) } - - /// 🆕 检查是否需要重新生成层级文件 - /// + + /// 检查是否需要重新生成层级文件 + /// /// 检查逻辑： /// 1. 如果 .abstract.md 或 .overview.md 不存在 → 需要生成 /// 2. 如果目录中有文件比 .abstract.md 更新 → 需要重新生成 @@ -341,16 +353,16 @@ impl LayerGenerator { async fn should_regenerate(&self, uri: &str) -> Result { let abstract_path = format!("{}/.abstract.md", uri); let overview_path = format!("{}/.overview.md", uri); - + // 检查层级文件是否存在 let abstract_exists = self.filesystem.exists(&abstract_path).await?; let overview_exists = self.filesystem.exists(&overview_path).await?; - + if !abstract_exists || !overview_exists { debug!("层级文件缺失，需要生成: {}", uri); return Ok(true); } - + // 读取 .abstract.md 中的时间戳 let abstract_content = match self.filesystem.read(&abstract_path).await { Ok(content) => content, @@ -359,17 +371,17 @@ impl LayerGenerator { return Ok(true); } }; - + // 提取 "Added" 时间戳 let abstract_timestamp = self.extract_added_timestamp(&abstract_content); - + if abstract_timestamp.is_none() { debug!(".abstract.md 缺少时间戳，需要重新生成: {}", uri); return Ok(true); } - + let abstract_time = abstract_timestamp.unwrap(); - + // 检查目录中的文件是否有更新 let entries = self.filesystem.list(uri).await?; for entry in entries { @@ -377,7 +389,7 @@ impl LayerGenerator { if entry.name.starts_with('.') || entry.is_directory { continue; } - + // 只检查 .md 和 .txt 文件 if entry.name.ends_with(".md") || entry.name.ends_with(".txt") { // 读取文件内容，提取其中的时间戳（如果有） @@ -392,12 +404,12 @@ impl LayerGenerator { } } } - + debug!("目录内容未变更，无需重新生成: {}", uri); Ok(false) } - - /// 🆕 从内容中提取 "Added" 时间戳 + + /// 从内容中提取 "Added" 时间戳 fn extract_added_timestamp(&self, content: &str) -> Option> { // 查找 "**Added**: YYYY-MM-DD HH:MM:SS UTC" 格式 if let Some(start) = content.find("**Added**: ") { @@ -412,18 +424,18 @@ impl LayerGenerator { } None } - + /// 聚合目录内容 async fn aggregate_directory_content(&self, uri: &str) -> Result { let entries = self.filesystem.list(uri).await?; let mut content = String::new(); - + for entry in entries { // 跳过隐藏文件和目录 if entry.name.starts_with('.') || entry.is_directory { continue; } - + // 只读取文本文件 if entry.name.ends_with(".md") || entry.name.ends_with(".txt") { match self.filesystem.read(&entry.uri).await { @@ -437,26 +449,26 @@ impl LayerGenerator { } } } - + // 截断到合理长度（避免超出 LLM 上下文限制） let max_chars = 10000; if content.len() > max_chars { content.truncate(max_chars); content.push_str("\n\n[内容已截断...]"); } - + Ok(content) } - + /// 强制执行 Abstract 长度限制 fn enforce_abstract_limit(&self, text: String) -> Result { let mut result = text.trim().to_string(); let max_chars = self.config.abstract_config.max_chars; - + if result.len() <= max_chars { return Ok(result); } - + // 截断到最后一个句号/问号/叹号 if let Some(pos) = result[..max_chars] .rfind(|c| c == '。' || c == '.' || c == '?' || c == '!' || c == '！' || c == '？') @@ -466,19 +478,19 @@ impl LayerGenerator { result.truncate(max_chars - 3); result.push_str("..."); } - + Ok(result) } - + /// 强制执行 Overview 长度限制 fn enforce_overview_limit(&self, text: String) -> Result { let mut result = text.trim().to_string(); let max_chars = self.config.overview_config.max_chars; - + if result.len() <= max_chars { return Ok(result); } - + // 截断到最后一个段落 if let Some(pos) = result[..max_chars].rfind("\n\n") { result.truncate(pos); @@ -487,33 +499,37 @@ impl LayerGenerator { result.truncate(max_chars - 3); result.push_str("..."); } - + Ok(result) } - + /// 重新生成所有超大的 .abstract 文件 pub async fn regenerate_oversized_abstracts(&self) -> Result { info!("扫描超大的 .abstract 文件..."); let directories = self.scan_all_directories().await?; let max_chars = self.config.abstract_config.max_chars; - + let mut stats = RegenerationStats { total: 0, regenerated: 0, failed: 0, }; - + for dir in directories { let abstract_path = format!("{}/.abstract.md", dir); - + if let Ok(content) = self.filesystem.read(&abstract_path).await { // 移除 "Added" 标记后再检查长度 let content_without_metadata = self.strip_metadata(&content); - + if content_without_metadata.len() > max_chars { stats.total += 1; - info!("发现超大 .abstract: {} ({} 字符)", dir, content_without_metadata.len()); - + info!( + "发现超大 .abstract: {} ({} 字符)", + dir, + content_without_metadata.len() + ); + match self.generate_layers_for_directory(&dir).await { Ok(_) => { stats.regenerated += 1; @@ -527,26 +543,26 @@ impl LayerGenerator { } } } - + info!( "重新生成完成: 总计 {}, 成功 {}, 失败 {}", stats.total, stats.regenerated, stats.failed ); - + Ok(stats) } - + /// 移除元数据（Added、Confidence等） fn strip_metadata(&self, content: &str) -> String { let mut result = content.to_string(); - + // 移除 **Added**: ... 行 if let Some(pos) = result.find("\n\n**Added**:") { result.truncate(pos); } else if let Some(pos) = result.find("**Added**:") { result.truncate(pos); } - + result.trim().to_string() } } diff --git a/cortex-mem-core/src/automation/manager.rs b/cortex-mem-core/src/automation/manager.rs index c4b0f6b..1b36ad8 100644 --- a/cortex-mem-core/src/automation/manager.rs +++ b/cortex-mem-core/src/automation/manager.rs @@ -1,7 +1,7 @@ use crate::{ + Result, automation::{AutoExtractor, AutoIndexer, LayerGenerator}, events::{CortexEvent, SessionEvent}, - Result, }; use std::collections::HashSet; use std::sync::Arc; @@ -22,9 +22,9 @@ pub struct AutomationConfig { pub index_on_close: bool, /// 索引批处理延迟（秒） pub index_batch_delay: u64, - /// 🆕 启动时自动生成缺失的 L0/L1 文件 + /// 启动时自动生成缺失的 L0/L1 文件 pub auto_generate_layers_on_startup: bool, - /// 🆕 每N条消息触发一次L0/L1生成（0表示禁用） + /// 每N条消息触发一次L0/L1生成（0表示禁用） pub generate_layers_every_n_messages: usize, } @@ -33,11 +33,11 @@ impl Default for AutomationConfig { Self { auto_index: true, auto_extract: true, - index_on_message: false, // 默认不实时索引（性能考虑） - index_on_close: true, // 默认会话关闭时索引 + index_on_message: false, // 默认不实时索引（性能考虑） + index_on_close: true, // 默认会话关闭时索引 index_batch_delay: 2, - auto_generate_layers_on_startup: false, // 🆕 默认关闭（避免启动时阻塞） - generate_layers_every_n_messages: 0, // 🆕 默认禁用（避免频繁LLM调用） + auto_generate_layers_on_startup: false, // 默认关闭（避免启动时阻塞） + generate_layers_every_n_messages: 0, // 默认禁用（避免频繁LLM调用） } } } @@ -46,7 +46,7 @@ impl Default for AutomationConfig { pub struct AutomationManager { indexer: Arc, extractor: Option>, - layer_generator: Option>, // 🆕 层级生成器 + layer_generator: Option>, // 层级生成器 config: AutomationConfig, } @@ -60,22 +60,22 @@ impl AutomationManager { Self { indexer, extractor, - layer_generator: None, // 🆕 初始为 None，需要单独设置 + layer_generator: None, // 初始为 None，需要单独设置 config, } } - - /// 🆕 设置层级生成器（可选） + + /// 设置层级生成器（可选） pub fn with_layer_generator(mut self, layer_generator: Arc) -> Self { self.layer_generator = Some(layer_generator); self } - + /// 🎯 核心方法：启动自动化任务 pub async fn start(self, mut event_rx: mpsc::UnboundedReceiver) -> Result<()> { info!("Starting AutomationManager with config: {:?}", self.config); - - // 🆕 启动时自动生成缺失的 L0/L1 文件 + + // 启动时自动生成缺失的 L0/L1 文件 if self.config.auto_generate_layers_on_startup { if let Some(ref generator) = self.layer_generator { info!("启动时检查并生成缺失的 L0/L1 文件..."); @@ -97,30 +97,31 @@ impl AutomationManager { warn!("auto_generate_layers_on_startup 已启用但未设置 layer_generator"); } } - + // 批处理缓冲区（收集需要索引的session_id） let mut pending_sessions: HashSet = HashSet::new(); let batch_delay = Duration::from_secs(self.config.index_batch_delay); let mut batch_timer: Option = None; - - // 🆕 会话消息计数器（用于触发定期L0/L1生成） - let mut session_message_counts: std::collections::HashMap = std::collections::HashMap::new(); - + + // 会话消息计数器（用于触发定期L0/L1生成） + let mut session_message_counts: std::collections::HashMap = + std::collections::HashMap::new(); + loop { tokio::select! { // 事件处理 Some(event) = event_rx.recv() => { if let Err(e) = self.handle_event( - event, - &mut pending_sessions, - &mut batch_timer, + event, + &mut pending_sessions, + &mut batch_timer, batch_delay, &mut session_message_counts ).await { warn!("Failed to handle event: {}", e); } } - + // 批处理定时器触发 _ = async { if let Some(deadline) = batch_timer { @@ -139,7 +140,7 @@ impl AutomationManager { } } } - + /// 处理事件 async fn handle_event( &self, @@ -152,36 +153,42 @@ impl AutomationManager { match event { CortexEvent::Session(SessionEvent::MessageAdded { session_id, .. }) => { // 更新消息计数 - let count = session_message_counts.entry(session_id.clone()).or_insert(0); + let count = session_message_counts + .entry(session_id.clone()) + .or_insert(0); *count += 1; - - // 🆕 检查是否需要基于消息数量触发L0/L1生成 - if self.config.generate_layers_every_n_messages > 0 - && *count % self.config.generate_layers_every_n_messages == 0 + + // 检查是否需要基于消息数量触发L0/L1生成 + if self.config.generate_layers_every_n_messages > 0 + && *count % self.config.generate_layers_every_n_messages == 0 { if let Some(ref generator) = self.layer_generator { info!( "Message count threshold reached ({} messages), triggering L0/L1 generation for session: {}", count, session_id ); - + // 异步生成L0/L1（避免阻塞） let generator_clone = generator.clone(); let indexer_clone = self.indexer.clone(); let session_id_clone = session_id.clone(); let auto_index = self.config.auto_index; - + tokio::spawn(async move { - let timeline_uri = format!("cortex://session/{}/timeline", session_id_clone); - + let timeline_uri = + format!("cortex://session/{}/timeline", session_id_clone); + // 生成L0/L1 match generator_clone.ensure_timeline_layers(&timeline_uri).await { Ok(stats) => { info!( "✓ Periodic L0/L1 generation for {}: total={}, generated={}, failed={}", - session_id_clone, stats.total, stats.generated, stats.failed + session_id_clone, + stats.total, + stats.generated, + stats.failed ); - + // 生成后索引（如果启用了auto_index） if auto_index && stats.generated > 0 { match indexer_clone.index_thread(&session_id_clone).await { @@ -192,19 +199,25 @@ impl AutomationManager { ); } Err(e) => { - warn!("✗ Failed to index L0/L1 for {}: {}", session_id_clone, e); + warn!( + "✗ Failed to index L0/L1 for {}: {}", + session_id_clone, e + ); } } } } Err(e) => { - warn!("✗ Periodic L0/L1 generation failed for {}: {}", session_id_clone, e); + warn!( + "✗ Periodic L0/L1 generation failed for {}: {}", + session_id_clone, e + ); } } }); } } - + if self.config.index_on_message { // 实时索引模式：立即索引 info!("Real-time indexing session: {}", session_id); @@ -212,18 +225,21 @@ impl AutomationManager { } else { // 批处理模式：加入待处理队列 pending_sessions.insert(session_id); - + // 启动批处理定时器（如果未启动） if batch_timer.is_none() { *batch_timer = Some(tokio::time::Instant::now() + batch_delay); } } } - + CortexEvent::Session(SessionEvent::Closed { session_id }) => { if self.config.index_on_close { - info!("Session closed, triggering async full processing: {}", session_id); - + info!( + "Session closed, triggering async full processing: {}", + session_id + ); + // 🔧 异步执行所有后处理任务，避免阻塞事件循环 let extractor = self.extractor.clone(); let generator = self.layer_generator.clone(); @@ -231,49 +247,65 @@ impl AutomationManager { let auto_extract = self.config.auto_extract; let auto_index = self.config.auto_index; let session_id_clone = session_id.clone(); - + tokio::spawn(async move { let start = tokio::time::Instant::now(); - + // 1. 自动提取记忆（如果配置了且有extractor） if auto_extract { if let Some(ref extractor) = extractor { match extractor.extract_session(&session_id_clone).await { Ok(stats) => { - info!("✓ Extraction completed for {}: {:?}", session_id_clone, stats); + info!( + "✓ Extraction completed for {}: {:?}", + session_id_clone, stats + ); } Err(e) => { - warn!("✗ Extraction failed for {}: {}", session_id_clone, e); + warn!( + "✗ Extraction failed for {}: {}", + session_id_clone, e + ); } } } } - + // 2. 生成 L0/L1 层级文件（如果配置了layer_generator） if let Some(ref generator) = generator { info!("Generating L0/L1 layers for session: {}", session_id_clone); - let timeline_uri = format!("cortex://session/{}/timeline", session_id_clone); - + let timeline_uri = + format!("cortex://session/{}/timeline", session_id_clone); + match generator.ensure_timeline_layers(&timeline_uri).await { Ok(stats) => { info!( "✓ L0/L1 generation completed for {}: total={}, generated={}, failed={}", - session_id_clone, stats.total, stats.generated, stats.failed + session_id_clone, + stats.total, + stats.generated, + stats.failed ); } Err(e) => { - warn!("✗ L0/L1 generation failed for {}: {}", session_id_clone, e); + warn!( + "✗ L0/L1 generation failed for {}: {}", + session_id_clone, e + ); } } } - + // 3. 索引整个会话（包括新生成的L0/L1/L2） if auto_index { match indexer.index_thread(&session_id_clone).await { Ok(stats) => { info!( "✓ Session {} indexed: {} indexed, {} skipped, {} errors", - session_id_clone, stats.total_indexed, stats.total_skipped, stats.total_errors + session_id_clone, + stats.total_indexed, + stats.total_skipped, + stats.total_errors ); } Err(e) => { @@ -281,7 +313,7 @@ impl AutomationManager { } } } - + let duration = start.elapsed(); info!( "🎉 Session {} post-processing completed in {:.2}s", @@ -289,30 +321,33 @@ impl AutomationManager { duration.as_secs_f64() ); }); - - info!("Session {} close acknowledged, post-processing running in background", session_id); + + info!( + "Session {} close acknowledged, post-processing running in background", + session_id + ); } } - + _ => { /* 其他事件暂时忽略 */ } } - + Ok(()) } - + /// 批量处理待索引的会话 async fn flush_batch(&self, pending_sessions: &mut HashSet) -> Result<()> { info!("Flushing batch: {} sessions", pending_sessions.len()); - + for session_id in pending_sessions.drain() { if let Err(e) = self.index_session(&session_id).await { warn!("Failed to index session {}: {}", session_id, e); } } - + Ok(()) } - + /// 索引单个会话 async fn index_session(&self, session_id: &str) -> Result<()> { match self.indexer.index_thread(session_id).await { diff --git a/cortex-mem-core/src/automation/mod.rs b/cortex-mem-core/src/automation/mod.rs index 3808528..db0c295 100644 --- a/cortex-mem-core/src/automation/mod.rs +++ b/cortex-mem-core/src/automation/mod.rs @@ -1,7 +1,7 @@ mod auto_extract; mod indexer; -mod layer_generator; // 🆕 层级生成器 -mod manager; // 🆕 自动化管理器 +mod layer_generator; +mod manager; mod sync; mod watcher; @@ -11,7 +11,10 @@ mod layer_generator_tests; pub use auto_extract::{AutoExtractConfig, AutoExtractStats, AutoExtractor, AutoSessionManager}; pub use indexer::{AutoIndexer, IndexStats, IndexerConfig}; -pub use layer_generator::{LayerGenerator, LayerGenerationConfig, GenerationStats, RegenerationStats, AbstractConfig, OverviewConfig}; // 🆕 导出 -pub use manager::{AutomationConfig, AutomationManager}; // 🆕 导出 +pub use layer_generator::{ + AbstractConfig, GenerationStats, LayerGenerationConfig, LayerGenerator, OverviewConfig, + RegenerationStats, +}; +pub use manager::{AutomationConfig, AutomationManager}; pub use sync::{SyncConfig, SyncManager, SyncStats}; pub use watcher::{FsEvent, FsWatcher, WatcherConfig}; diff --git a/cortex-mem-core/src/automation/sync.rs b/cortex-mem-core/src/automation/sync.rs index c9c8cde..ad48467 100644 --- a/cortex-mem-core/src/automation/sync.rs +++ b/cortex-mem-core/src/automation/sync.rs @@ -145,6 +145,45 @@ impl SyncManager { Ok(total_stats) } + /// 同步特定路径到向量数据库 + /// + /// 用于只索引特定session或特定路径的文件 + /// 例如: sync_specific_path("cortex://session/abc123") + pub async fn sync_specific_path(&self, uri: &str) -> Result { + info!("Starting sync for specific path: {}", uri); + + // 检查路径是否存在 + if !self.filesystem.exists(uri).await? { + warn!("Path does not exist: {}", uri); + return Ok(SyncStats::default()); + } + + // 判断是session路径还是其他路径 + let stats = if uri.starts_with("cortex://session/") { + // session路径使用递归同步（包含timeline等子目录） + self.sync_directory_recursive(uri).await? + } else if uri.starts_with("cortex://user/") || uri.starts_with("cortex://agent/") { + // user/agent路径使用非递归同步 + self.sync_directory(uri, MemoryType::Semantic).await? + } else if uri.starts_with("cortex://resources/") { + self.sync_directory(uri, MemoryType::Semantic).await? + } else { + // 其他路径尝试递归同步 + self.sync_directory_recursive(uri).await? + }; + + info!( + "Sync completed for {}: {} files processed, {} indexed, {} skipped, {} errors", + uri, + stats.total_files, + stats.indexed_files, + stats.skipped_files, + stats.error_files + ); + + Ok(stats) + } + /// 同步单个目录（非递归） fn sync_directory<'a>( &'a self, diff --git a/cortex-mem-core/src/builder.rs b/cortex-mem-core/src/builder.rs index d2ba8fb..e3ffe7d 100644 --- a/cortex-mem-core/src/builder.rs +++ b/cortex-mem-core/src/builder.rs @@ -56,7 +56,7 @@ impl CortexMemBuilder { self } - /// 🆕 配置自动化行为 + /// 配置自动化行为 pub fn with_automation(mut self, config: AutomationConfig) -> Self { self.automation_config = config; self diff --git a/cortex-mem-core/src/config.rs b/cortex-mem-core/src/config.rs index 2bdf6d2..750400a 100644 --- a/cortex-mem-core/src/config.rs +++ b/cortex-mem-core/src/config.rs @@ -7,7 +7,7 @@ pub struct QdrantConfig { pub collection_name: String, pub embedding_dim: Option, pub timeout_secs: u64, - /// 🆕 Optional tenant ID for collection isolation + /// Optional tenant ID for collection isolation /// If set, collection_name will be suffixed with "_" pub tenant_id: Option, } @@ -19,13 +19,13 @@ impl Default for QdrantConfig { collection_name: "cortex-mem".to_string(), embedding_dim: None, timeout_secs: 30, - tenant_id: None, // 🆕 默认不使用租户隔离 + tenant_id: None, } } } impl QdrantConfig { - /// 🆕 Get the actual collection name with tenant isolation + /// Get the actual collection name with tenant isolation pub fn get_collection_name(&self) -> String { if let Some(tenant_id) = &self.tenant_id { format!("{}_{}", self.collection_name, tenant_id) @@ -33,8 +33,8 @@ impl QdrantConfig { self.collection_name.clone() } } - - /// 🆕 Create a new config with tenant ID + + /// Create a new config with tenant ID pub fn with_tenant(mut self, tenant_id: impl Into) -> Self { self.tenant_id = Some(tenant_id.into()); self diff --git a/cortex-mem-core/src/embedding/mod.rs b/cortex-mem-core/src/embedding/mod.rs index a244f0b..219a2da 100644 --- a/cortex-mem-core/src/embedding/mod.rs +++ b/cortex-mem-core/src/embedding/mod.rs @@ -1,5 +1,5 @@ -mod client; -mod cache; // 🆕 Embedding 缓存层 +mod cache; +mod client; // Embedding 缓存层 +pub use cache::{CacheConfig, CacheStats, EmbeddingCache, EmbeddingProvider}; pub use client::{EmbeddingClient, EmbeddingConfig}; -pub use cache::{EmbeddingCache, CacheConfig, CacheStats, EmbeddingProvider}; // 🆕 diff --git a/cortex-mem-core/src/filesystem/operations.rs b/cortex-mem-core/src/filesystem/operations.rs index 6f585ca..8a0ec83 100644 --- a/cortex-mem-core/src/filesystem/operations.rs +++ b/cortex-mem-core/src/filesystem/operations.rs @@ -11,19 +11,19 @@ use super::uri::UriParser; pub trait FilesystemOperations: Send + Sync { /// List directory contents async fn list(&self, uri: &str) -> Result>; - + /// Read file content async fn read(&self, uri: &str) -> Result; - + /// Write file content async fn write(&self, uri: &str, content: &str) -> Result<()>; - + /// Delete file or directory async fn delete(&self, uri: &str) -> Result<()>; - + /// Check if file/directory exists async fn exists(&self, uri: &str) -> Result; - + /// Get file metadata async fn metadata(&self, uri: &str) -> Result; } @@ -42,7 +42,7 @@ impl CortexFilesystem { tenant_id: None, } } - + /// Create a new CortexFilesystem with tenant isolation pub fn with_tenant(root: impl AsRef, tenant_id: impl Into) -> Self { Self { @@ -50,22 +50,22 @@ impl CortexFilesystem { tenant_id: Some(tenant_id.into()), } } - + /// Get the root path pub fn root_path(&self) -> &Path { &self.root } - + /// Get the tenant ID pub fn tenant_id(&self) -> Option<&str> { self.tenant_id.as_deref() } - + /// Set the tenant ID dynamically (for runtime tenant switching) pub fn set_tenant(&mut self, tenant_id: Option>) { self.tenant_id = tenant_id.map(|id| id.into()); } - + /// Initialize the filesystem structure pub async fn initialize(&self) -> Result<()> { // Get the base directory (with or without tenant) @@ -76,11 +76,11 @@ impl CortexFilesystem { // For non-tenant: /root/ self.root.clone() }; - + // Create root directory fs::create_dir_all(&base_dir).await?; - - // 🆕 只有在tenant模式下才创建维度目录 + + // 只有在tenant模式下才创建维度目录 // Non-tenant模式（如cortex-mem-service全局实例）不应创建这些目录 if self.tenant_id.is_some() { // Create dimension directories (OpenViking style: resources, user, agent, session) @@ -89,14 +89,14 @@ impl CortexFilesystem { fs::create_dir_all(dir).await?; } } - + Ok(()) } - + /// Get file path from URI (with tenant isolation) fn uri_to_path(&self, uri: &str) -> Result { let parsed_uri = UriParser::parse(uri)?; - + // If tenant_id exists, add tenant prefix (without extra cortex subfolder) let path = if let Some(tenant_id) = &self.tenant_id { // /root/tenants/{tenant_id}/{path} @@ -106,10 +106,10 @@ impl CortexFilesystem { // /root/{path} parsed_uri.to_file_path(&self.root) }; - + Ok(path) } - + /// Load metadata from .metadata.json #[allow(dead_code)] async fn load_metadata(&self, dir_path: &Path) -> Result> { @@ -117,12 +117,12 @@ impl CortexFilesystem { if !metadata_path.try_exists()? { return Ok(None); } - + let content = fs::read_to_string(metadata_path).await?; let metadata: MemoryMetadata = serde_json::from_str(&content)?; Ok(Some(metadata)) } - + /// Save metadata to .metadata.json #[allow(dead_code)] async fn save_metadata(&self, dir_path: &Path, metadata: &MemoryMetadata) -> Result<()> { @@ -137,108 +137,108 @@ impl CortexFilesystem { impl FilesystemOperations for CortexFilesystem { async fn list(&self, uri: &str) -> Result> { let path = self.uri_to_path(uri)?; - + if !path.try_exists()? { return Err(Error::NotFound { uri: uri.to_string(), }); } - + let mut entries = Vec::new(); let mut read_dir = fs::read_dir(&path).await?; - + while let Some(entry) = read_dir.next_entry().await? { let metadata = entry.metadata().await?; let name = entry.file_name().to_string_lossy().to_string(); - + // Skip hidden files except .abstract.md and .overview.md - if name.starts_with('.') - && name != ".abstract.md" - && name != ".overview.md" - { + if name.starts_with('.') && name != ".abstract.md" && name != ".overview.md" { continue; } - + let entry_uri = format!("{}/{}", uri.trim_end_matches('/'), name); - + entries.push(FileEntry { uri: entry_uri, name, is_directory: metadata.is_dir(), size: metadata.len(), - modified: metadata.modified() + modified: metadata + .modified() .map(|t| t.into()) .unwrap_or_else(|_| Utc::now()), }); } - + Ok(entries) } - + async fn read(&self, uri: &str) -> Result { let path = self.uri_to_path(uri)?; - + if !path.try_exists()? { return Err(Error::NotFound { uri: uri.to_string(), }); } - + let content = fs::read_to_string(&path).await?; Ok(content) } - + async fn write(&self, uri: &str, content: &str) -> Result<()> { let path = self.uri_to_path(uri)?; - + // Create parent directories if let Some(parent) = path.parent() { fs::create_dir_all(parent).await?; } - + fs::write(&path, content).await?; Ok(()) } - + async fn delete(&self, uri: &str) -> Result<()> { let path = self.uri_to_path(uri)?; - + if !path.try_exists()? { return Err(Error::NotFound { uri: uri.to_string(), }); } - + if path.is_dir() { fs::remove_dir_all(&path).await?; } else { fs::remove_file(&path).await?; } - + Ok(()) } - + async fn exists(&self, uri: &str) -> Result { let path = self.uri_to_path(uri)?; Ok(path.try_exists().unwrap_or(false)) } - + async fn metadata(&self, uri: &str) -> Result { let path = self.uri_to_path(uri)?; - + if !path.try_exists()? { return Err(Error::NotFound { uri: uri.to_string(), }); } - + let metadata = fs::metadata(&path).await?; - + Ok(FileMetadata { - created_at: metadata.created() + created_at: metadata + .created() .map(|t| t.into()) .unwrap_or_else(|_| Utc::now()), - updated_at: metadata.modified() + updated_at: metadata + .modified() .map(|t| t.into()) .unwrap_or_else(|_| Utc::now()), size: metadata.len(), diff --git a/cortex-mem-core/src/layers/mod.rs b/cortex-mem-core/src/layers/mod.rs index f4a21d6..602274a 100644 --- a/cortex-mem-core/src/layers/mod.rs +++ b/cortex-mem-core/src/layers/mod.rs @@ -1,3 +1,3 @@ pub mod generator; pub mod manager; -pub mod reader; // 🆕 并发层级读取器 +pub mod reader; diff --git a/cortex-mem-core/src/lib.rs b/cortex-mem-core/src/lib.rs index c64b298..74eaf5e 100644 --- a/cortex-mem-core/src/lib.rs +++ b/cortex-mem-core/src/lib.rs @@ -46,40 +46,43 @@ pub mod config; pub mod error; -pub mod events; // 🆕 事件系统 +pub mod events; pub mod logging; pub mod types; pub mod automation; +pub mod builder; +pub mod embedding; pub mod extraction; pub mod filesystem; -pub mod builder; // 🆕 统一初始化API pub mod layers; pub mod llm; pub mod search; pub mod session; pub mod vector_store; -pub mod embedding; // Re-exports pub use config::*; pub use error::{Error, Result}; -pub use events::{CortexEvent, EventBus, FilesystemEvent, SessionEvent}; // 🆕 导出事件类型 +pub use events::{CortexEvent, EventBus, FilesystemEvent, SessionEvent}; pub use types::*; -pub use automation::{AutoExtractConfig, AutoExtractor, AutoIndexer, AutomationConfig, AutomationManager, FsWatcher, IndexStats, IndexerConfig, SyncConfig, SyncManager, SyncStats, WatcherConfig}; -pub use builder::{CortexMem, CortexMemBuilder}; // 🆕 导出统一API +pub use automation::{ + AutoExtractConfig, AutoExtractor, AutoIndexer, AutomationConfig, AutomationManager, FsWatcher, + IndexStats, IndexerConfig, SyncConfig, SyncManager, SyncStats, WatcherConfig, +}; +pub use builder::{CortexMem, CortexMemBuilder}; pub use extraction::ExtractionConfig; // Note: MemoryExtractor is also exported from session module +pub use embedding::{EmbeddingClient, EmbeddingConfig}; pub use filesystem::{CortexFilesystem, FilesystemOperations}; pub use llm::LLMClient; pub use search::{SearchOptions, VectorSearchEngine}; pub use session::{ - Message, MessageRole, Participant, ParticipantManager, SessionConfig, SessionManager, - MemoryExtractor, ExtractedMemories, PreferenceMemory, EntityMemory, EventMemory, CaseMemory, + CaseMemory, EntityMemory, EventMemory, ExtractedMemories, MemoryExtractor, Message, + MessageRole, Participant, ParticipantManager, PreferenceMemory, SessionConfig, SessionManager, }; -pub use vector_store::{QdrantVectorStore, VectorStore, uri_to_vector_id, parse_vector_id}; -pub use embedding::{EmbeddingClient, EmbeddingConfig}; +pub use vector_store::{QdrantVectorStore, VectorStore, parse_vector_id, uri_to_vector_id}; // Session-related re-exports pub use session::message::MessageStorage; diff --git a/cortex-mem-core/src/search/vector_engine.rs b/cortex-mem-core/src/search/vector_engine.rs index a9b982c..d51f6d0 100644 --- a/cortex-mem-core/src/search/vector_engine.rs +++ b/cortex-mem-core/src/search/vector_engine.rs @@ -123,13 +123,34 @@ impl VectorSearchEngine { let query_vec = self.embedding.embed(query).await?; // 2. Search in Qdrant - let filters = crate::types::Filters::default(); + // ✅ 修复：构建包含scope的Filters + let mut filters = crate::types::Filters::default(); + if let Some(scope) = &options.root_uri { + filters.uri_prefix = Some(scope.clone()); + } + let scored = self .qdrant .as_ref() .search_with_threshold(&query_vec, &filters, options.limit, Some(options.threshold)) .await?; + // ✅ 修复：添加应用层URI前缀过滤（确保scope隔离） + let scope_prefix = options.root_uri.as_ref(); + let scored: Vec<_> = scored + .into_iter() + .filter(|result| { + if let Some(prefix) = scope_prefix { + if let Some(uri) = &result.memory.metadata.uri { + return uri.starts_with(prefix); + } + // 如果没有URI metadata，保守地排除（防止泄露） + return false; + } + true + }) + .collect(); + // 3. Enrich results with content let mut results = Vec::new(); for scored_mem in scored { @@ -191,7 +212,7 @@ impl VectorSearchEngine { intent.intent_type, intent.keywords ); - // 🆕 自适应阈值：根据查询类型动态调整 + // 自适应阈值：根据查询类型动态调整 let adaptive_threshold = Self::adaptive_l0_threshold(query, &intent.intent_type); // Generate query embedding once (use rewritten query if available) @@ -233,18 +254,18 @@ impl VectorSearchEngine { }) .collect(); - // 🆕 增强降级检索策略 + // 增强降级检索策略 if l0_results.is_empty() { warn!( "No L0 results found at threshold {}, trying fallback strategies", adaptive_threshold ); - // 策略1: 降低阈值重试（对于实体查询可能已经是0.4了，尝试更低） + // 策略1: 降低阈值重试（但不要降得太低，防止返回过多不相关结果） let relaxed_threshold = if adaptive_threshold <= 0.4 { - 0.3 // 对于已经很低的阈值，尝试0.3 + 0.4 // 最低不低于0.4（余弦相似度约60度） } else { - adaptive_threshold - 0.3 // 否则降低0.3 + (adaptive_threshold - 0.2).max(0.4) // 降低0.2，但最低0.4 }; info!( @@ -287,7 +308,11 @@ impl VectorSearchEngine { } else { // 策略2: 完全降级到语义搜索（跳过L0，直接全量L2检索） warn!( - "No results even with relaxed threshold, falling back to full semantic search" + "No results even with relaxed threshold {}, falling back to full semantic search", + relaxed_threshold + ); + warn!( + "⚠️ Semantic search fallback may return less relevant results due to lack of L0/L1 guidance" ); return self.semantic_search(query, options).await; } @@ -302,7 +327,7 @@ impl VectorSearchEngine { .await } - /// 🆕 继续执行分层检索的L1/L2阶段 + /// 继续执行分层检索的L1/L2阶段 /// /// 这个方法被提取出来，以便在降级重试后复用 async fn continue_layered_search( @@ -619,7 +644,7 @@ impl VectorSearchEngine { QueryIntentType::General } - /// 🆕 判断查询是否可能是实体查询（人名、地名、组织名等） + /// 判断查询是否可能是实体查询（人名、地名、组织名等） /// /// 实体查询的特征： /// - 查询很短（通常2-4个字符/词） @@ -665,7 +690,7 @@ impl VectorSearchEngine { false } - /// 🆕 根据查询意图自适应计算L0阈值 + /// 根据查询意图自适应计算L0阈值 /// /// 不同查询类型使用不同阈值： /// - 实体查询: 0.4 (降低阈值，因为L0摘要可能丢失实体) diff --git a/cortex-mem-core/src/session/extraction.rs b/cortex-mem-core/src/session/extraction.rs index e6b4863..71e732c 100644 --- a/cortex-mem-core/src/session/extraction.rs +++ b/cortex-mem-core/src/session/extraction.rs @@ -25,16 +25,16 @@ pub struct ExtractedMemories { /// Agent cases (problem + solution) #[serde(default)] pub cases: Vec, - /// 🆕 Personal information (age, occupation, education, etc.) + /// Personal information (age, occupation, education, etc.) #[serde(default)] pub personal_info: Vec, - /// 🆕 Work history (companies, roles, durations) + /// Work history (companies, roles, durations) #[serde(default)] pub work_history: Vec, - /// 🆕 Relationships (family, friends, colleagues) + /// Relationships (family, friends, colleagues) #[serde(default)] pub relationships: Vec, - /// 🆕 Goals (career goals, personal goals) + /// Goals (career goals, personal goals) #[serde(default)] pub goals: Vec, } @@ -89,7 +89,7 @@ pub struct CaseMemory { pub lessons_learned: Vec, } -/// 🆕 Personal information memory +/// Personal information memory #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PersonalInfoMemory { pub category: String, // e.g., "age", "occupation", "education", "location" @@ -97,7 +97,7 @@ pub struct PersonalInfoMemory { pub confidence: f32, } -/// 🆕 Work history memory +/// Work history memory #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WorkHistoryMemory { pub company: String, @@ -107,7 +107,7 @@ pub struct WorkHistoryMemory { pub confidence: f32, } -/// 🆕 Relationship memory +/// Relationship memory #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RelationshipMemory { pub person: String, @@ -116,7 +116,7 @@ pub struct RelationshipMemory { pub confidence: f32, } -/// 🆕 Goal memory +/// Goal memory #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GoalMemory { pub goal: String, @@ -129,8 +129,8 @@ pub struct GoalMemory { pub struct MemoryExtractor { llm_client: Arc, filesystem: Arc, - user_id: String, // 🆕 用户ID用于记忆隔离 - agent_id: String, // 🆕 Agent ID用于记忆隔离 + user_id: String, + agent_id: String, } impl MemoryExtractor { @@ -368,7 +368,7 @@ Return ONLY the JSON object. No additional text before or after."#, self.filesystem.write(&uri, &content).await?; } - // 🆕 Save personal info with deduplication + // Save personal info with deduplication let personal_info_dir = format!("cortex://user/{}/personal_info", self.user_id); let existing_personal_info = self.load_existing_memories(&personal_info_dir).await?; let new_personal_info = @@ -387,7 +387,7 @@ Return ONLY the JSON object. No additional text before or after."#, self.filesystem.write(&uri, &content).await?; } - // 🆕 Save work history with deduplication + // Save work history with deduplication let work_history_dir = format!("cortex://user/{}/work_history", self.user_id); let existing_work_history = self.load_existing_memories(&work_history_dir).await?; let new_work_history = @@ -409,7 +409,7 @@ Return ONLY the JSON object. No additional text before or after."#, self.filesystem.write(&uri, &content).await?; } - // 🆕 Save relationships with deduplication + // Save relationships with deduplication let relationships_dir = format!("cortex://user/{}/relationships", self.user_id); let existing_relationships = self.load_existing_memories(&relationships_dir).await?; let new_relationships = @@ -429,7 +429,7 @@ Return ONLY the JSON object. No additional text before or after."#, self.filesystem.write(&uri, &content).await?; } - // 🆕 Save goals with deduplication + // Save goals with deduplication let goals_dir = format!("cortex://user/{}/goals", self.user_id); let existing_goals = self.load_existing_memories(&goals_dir).await?; let new_goals = self.deduplicate_goals(&memories.goals, &existing_goals); @@ -533,7 +533,7 @@ Return ONLY the JSON object. No additional text before or after."#, .collect() } - /// 🆕 Deduplicate personal info against existing ones + /// Deduplicate personal info against existing ones fn deduplicate_personal_info( &self, new_info: &[PersonalInfoMemory], @@ -553,7 +553,7 @@ Return ONLY the JSON object. No additional text before or after."#, .collect() } - /// 🆕 Deduplicate work history against existing ones + /// Deduplicate work history against existing ones fn deduplicate_work_history( &self, new_work: &[WorkHistoryMemory], @@ -574,7 +574,7 @@ Return ONLY the JSON object. No additional text before or after."#, .collect() } - /// 🆕 Deduplicate relationships against existing ones + /// Deduplicate relationships against existing ones fn deduplicate_relationships( &self, new_rels: &[RelationshipMemory], @@ -595,7 +595,7 @@ Return ONLY the JSON object. No additional text before or after."#, .collect() } - /// 🆕 Deduplicate goals against existing ones + /// Deduplicate goals against existing ones fn deduplicate_goals( &self, new_goals: &[GoalMemory], diff --git a/cortex-mem-core/src/session/manager.rs b/cortex-mem-core/src/session/manager.rs index 021482e..ca21a75 100644 --- a/cortex-mem-core/src/session/manager.rs +++ b/cortex-mem-core/src/session/manager.rs @@ -1,7 +1,7 @@ -use crate::{CortexFilesystem, FilesystemOperations, MessageStorage, ParticipantManager, Result}; +use crate::events::{CortexEvent, EventBus, SessionEvent}; use crate::llm::LLMClient; use crate::session::extraction::MemoryExtractor; -use crate::events::{EventBus, SessionEvent, CortexEvent}; // 🆕 导入事件类型 +use crate::{CortexFilesystem, FilesystemOperations, MessageStorage, ParticipantManager, Result}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -29,8 +29,8 @@ pub struct SessionMetadata { pub tags: Vec, pub title: Option, pub description: Option, - pub user_id: Option, // 🆕 用户ID用于记忆隔离 - pub agent_id: Option, // 🆕 Agent ID用于记忆隔离 + pub user_id: Option, + pub agent_id: Option, } impl SessionMetadata { @@ -52,7 +52,7 @@ impl SessionMetadata { agent_id: None, } } - + /// Create new session metadata with user_id and agent_id pub fn with_ids( thread_id: impl Into, @@ -64,26 +64,26 @@ impl SessionMetadata { metadata.agent_id = agent_id; metadata } - + /// Mark session as closed pub fn close(&mut self) { self.status = SessionStatus::Closed; self.closed_at = Some(Utc::now()); self.updated_at = Utc::now(); } - + /// Mark session as archived pub fn archive(&mut self) { self.status = SessionStatus::Archived; self.updated_at = Utc::now(); } - + /// Update message count pub fn update_message_count(&mut self, count: usize) { self.message_count = count; self.updated_at = Utc::now(); } - + /// Add a participant pub fn add_participant(&mut self, participant_id: impl Into) { let id = participant_id.into(); @@ -92,7 +92,7 @@ impl SessionMetadata { self.updated_at = Utc::now(); } } - + /// Add a tag pub fn add_tag(&mut self, tag: impl Into) { let t = tag.into(); @@ -101,47 +101,56 @@ impl SessionMetadata { self.updated_at = Utc::now(); } } - + /// Set title pub fn set_title(&mut self, title: impl Into) { self.title = Some(title.into()); self.updated_at = Utc::now(); } - + /// Convert to markdown pub fn to_markdown(&self) -> String { let mut md = String::new(); - + md.push_str(&format!("# Session: {}\n\n", self.thread_id)); - + if let Some(ref title) = self.title { md.push_str(&format!("**Title**: {}\n\n", title)); } - + md.push_str(&format!("**Status**: {:?}\n", self.status)); - md.push_str(&format!("**Created**: {}\n", self.created_at.format("%Y-%m-%d %H:%M:%S UTC"))); - md.push_str(&format!("**Updated**: {}\n", self.updated_at.format("%Y-%m-%d %H:%M:%S UTC"))); - + md.push_str(&format!( + "**Created**: {}\n", + self.created_at.format("%Y-%m-%d %H:%M:%S UTC") + )); + md.push_str(&format!( + "**Updated**: {}\n", + self.updated_at.format("%Y-%m-%d %H:%M:%S UTC") + )); + if let Some(closed_at) = self.closed_at { - md.push_str(&format!("**Closed**: {}\n", closed_at.format("%Y-%m-%d %H:%M:%S UTC"))); + md.push_str(&format!( + "**Closed**: {}\n", + closed_at.format("%Y-%m-%d %H:%M:%S UTC") + )); } - + md.push_str(&format!("**Messages**: {}\n", self.message_count)); md.push_str(&format!("**Participants**: {}\n", self.participants.len())); - + if !self.tags.is_empty() { md.push_str(&format!("**Tags**: {}\n", self.tags.join(", "))); } - + if let Some(ref description) = self.description { md.push_str(&format!("\n## Description\n\n{}\n", description)); } - + md.push_str("\n## Participants\n\n"); for participant in &self.participants { md.push_str(&format!("- {}\n", participant)); } - + md } } @@ -171,10 +180,10 @@ pub struct ExtractionStats { pub entities: usize, pub events: usize, pub cases: usize, - pub personal_info: usize, // 🆕 个人信息数量 - pub work_history: usize, // 🆕 工作经历数量 - pub relationships: usize, // 🆕 人际关系数量 - pub goals: usize, // 🆕 目标愿景数量 + pub personal_info: usize, + pub work_history: usize, + pub relationships: usize, + pub goals: usize, } /// Session manager @@ -184,7 +193,7 @@ pub struct SessionManager { participant_manager: ParticipantManager, config: SessionConfig, llm_client: Option>, - event_bus: Option, // 🆕 事件总线（可选） + event_bus: Option, } impl SessionManager { @@ -192,37 +201,37 @@ impl SessionManager { pub fn new(filesystem: Arc, config: SessionConfig) -> Self { let message_storage = MessageStorage::new(filesystem.clone()); let participant_manager = ParticipantManager::new(); - + Self { filesystem, message_storage, participant_manager, config, llm_client: None, - event_bus: None, // 🆕 默认不启用事件 + event_bus: None, } } - + /// Create a new session manager with LLM client for memory extraction pub fn new_with_llm( - filesystem: Arc, + filesystem: Arc, config: SessionConfig, - llm_client: Arc + llm_client: Arc, ) -> Self { let message_storage = MessageStorage::new(filesystem.clone()); let participant_manager = ParticipantManager::new(); - + Self { filesystem, message_storage, participant_manager, config, llm_client: Some(llm_client), - event_bus: None, // 🆕 默认不启用事件 + event_bus: None, } } - - /// 🆕 Create session manager with event bus for automation + + /// Create session manager with event bus for automation pub fn with_event_bus( filesystem: Arc, config: SessionConfig, @@ -230,7 +239,7 @@ impl SessionManager { ) -> Self { let message_storage = MessageStorage::new(filesystem.clone()); let participant_manager = ParticipantManager::new(); - + Self { filesystem, message_storage, @@ -240,8 +249,8 @@ impl SessionManager { event_bus: Some(event_bus), } } - - /// 🆕 Create session manager with LLM and event bus + + /// Create session manager with LLM and event bus pub fn with_llm_and_events( filesystem: Arc, config: SessionConfig, @@ -250,7 +259,7 @@ impl SessionManager { ) -> Self { let message_storage = MessageStorage::new(filesystem.clone()); let participant_manager = ParticipantManager::new(); - + Self { filesystem, message_storage, @@ -260,12 +269,12 @@ impl SessionManager { event_bus: Some(event_bus), } } - - /// 🆕 获取 LLM client（如果存在） + + /// 获取 LLM client（如果存在） pub fn llm_client(&self) -> Option<&Arc> { self.llm_client.as_ref() } - + /// Create a new session /// Create a new session with user_id and agent_id pub async fn create_session_with_ids( @@ -275,19 +284,19 @@ impl SessionManager { agent_id: Option, ) -> Result { let metadata = SessionMetadata::with_ids(thread_id, user_id, agent_id); - + // Save metadata to filesystem let metadata_uri = format!("cortex://session/{}/.session.json", thread_id); let metadata_json = serde_json::to_string_pretty(&metadata)?; self.filesystem.write(&metadata_uri, &metadata_json).await?; - - // 🆕 发布会话创建事件 + + // 发布会话创建事件 if let Some(ref bus) = self.event_bus { let _ = bus.publish(CortexEvent::Session(SessionEvent::Created { session_id: thread_id.to_string(), })); } - + Ok(metadata) } @@ -295,7 +304,7 @@ impl SessionManager { pub async fn create_session(&self, thread_id: &str) -> Result { self.create_session_with_ids(thread_id, None, None).await } - + /// Load session metadata pub async fn load_session(&self, thread_id: &str) -> Result { let metadata_uri = format!("cortex://session/{}/.session.json", thread_id); @@ -303,7 +312,7 @@ impl SessionManager { let metadata: SessionMetadata = serde_json::from_str(&metadata_json)?; Ok(metadata) } - + /// Update session metadata pub async fn update_session(&self, metadata: &SessionMetadata) -> Result<()> { let metadata_uri = format!("cortex://session/{}/.session.json", metadata.thread_id); @@ -311,37 +320,49 @@ impl SessionManager { self.filesystem.write(&metadata_uri, &metadata_json).await?; Ok(()) } - + /// Close a session pub async fn close_session(&mut self, thread_id: &str) -> Result { let mut metadata = self.load_session(thread_id).await?; metadata.close(); self.update_session(&metadata).await?; - - // 🆕 Generate timeline layers (L0/L1) for the entire session + + // Generate timeline layers (L0/L1) for the entire session if let Some(ref llm_client) = self.llm_client { use crate::layers::manager::LayerManager; - + let timeline_uri = format!("cortex://session/{}/timeline", thread_id); let layer_manager = LayerManager::new(self.filesystem.clone(), llm_client.clone()); - - info!("Generating session-level timeline layers for: {}", thread_id); + + info!( + "Generating session-level timeline layers for: {}", + thread_id + ); match layer_manager.generate_timeline_layers(&timeline_uri).await { Ok(_) => { - info!("✅ Successfully generated timeline layers for session: {}", thread_id); + info!( + "✅ Successfully generated timeline layers for session: {}", + thread_id + ); } Err(e) => { - warn!("Failed to generate timeline layers for session {}: {}", thread_id, e); + warn!( + "Failed to generate timeline layers for session {}: {}", + thread_id, e + ); } } } - + // Trigger memory extraction if auto_extract_on_close is enabled and LLM client is available if self.config.auto_extract_on_close { if let Some(ref llm_client) = self.llm_client { info!("Auto-extracting memories for session: {}", thread_id); - - match self.extract_and_save_memories(thread_id, llm_client.clone()).await { + + match self + .extract_and_save_memories(thread_id, llm_client.clone()) + .await + { Ok(stats) => { info!( "Memory extraction completed for session {}: {} preferences, {} entities, {} events, {} cases, {} personal_info, {} work_history, {} relationships, {} goals", @@ -357,43 +378,58 @@ impl SessionManager { ); } Err(e) => { - warn!("Failed to extract memories for session {}: {}", thread_id, e); + warn!( + "Failed to extract memories for session {}: {}", + thread_id, e + ); } } } else { - warn!("Memory extraction skipped for session {}: LLM client not configured", thread_id); + warn!( + "Memory extraction skipped for session {}: LLM client not configured", + thread_id + ); } } - - // 🆕 发布会话关闭事件 + + // 发布会话关闭事件 if let Some(ref bus) = self.event_bus { let _ = bus.publish(CortexEvent::Session(SessionEvent::Closed { session_id: thread_id.to_string(), })); } - + Ok(metadata) } - + /// Extract and save memories from a session async fn extract_and_save_memories( - &self, - thread_id: &str, - llm_client: Arc + &self, + thread_id: &str, + llm_client: Arc, ) -> Result { // Get all message URIs from the session timeline let message_uris = self.message_storage.list_messages(thread_id).await?; - + if message_uris.is_empty() { - info!("No messages found in session {}, skipping extraction", thread_id); + info!( + "No messages found in session {}, skipping extraction", + thread_id + ); return Ok(ExtractionStats::default()); } - + // 🔧 读取session metadata获取user_id和agent_id let metadata = self.load_session(thread_id).await?; - let user_id = metadata.user_id.clone().unwrap_or_else(|| "default".to_string()); - let agent_id = metadata.agent_id.clone().unwrap_or_else(|| "default".to_string()); - + let user_id = metadata + .user_id + .clone() + .unwrap_or_else(|| "default".to_string()); + let agent_id = metadata + .agent_id + .clone() + .unwrap_or_else(|| "default".to_string()); + // Read message contents let mut messages = Vec::new(); for uri in &message_uris { @@ -402,16 +438,12 @@ impl SessionManager { Err(e) => warn!("Failed to read message {}: {}", uri, e), } } - + // Extract memories using LLM - let extractor = MemoryExtractor::new( - llm_client, - self.filesystem.clone(), - user_id, - agent_id, - ); + let extractor = + MemoryExtractor::new(llm_client, self.filesystem.clone(), user_id, agent_id); let extracted = extractor.extract(&messages).await?; - + let stats = ExtractionStats { preferences: extracted.preferences.len(), entities: extracted.entities.len(), @@ -422,13 +454,13 @@ impl SessionManager { relationships: extracted.relationships.len(), goals: extracted.goals.len(), }; - + // Save extracted memories extractor.save_memories(&extracted).await?; - + Ok(stats) } - + /// Archive a session pub async fn archive_session(&self, thread_id: &str) -> Result { let mut metadata = self.load_session(thread_id).await?; @@ -436,30 +468,30 @@ impl SessionManager { self.update_session(&metadata).await?; Ok(metadata) } - + /// Delete a session pub async fn delete_session(&self, thread_id: &str) -> Result<()> { let session_uri = format!("cortex://session/{}", thread_id); self.filesystem.delete(&session_uri).await } - + /// Check if session exists pub async fn session_exists(&self, thread_id: &str) -> Result { let metadata_uri = format!("cortex://session/{}/.session.json", thread_id); self.filesystem.exists(&metadata_uri).await } - + /// Get message storage pub fn message_storage(&self) -> &MessageStorage { &self.message_storage } - + /// Get participant manager pub fn participant_manager(&mut self) -> &mut ParticipantManager { &mut self.participant_manager } - - /// 🆕 Add a message to a session (convenience method that also publishes events) + + /// Add a message to a session (convenience method that also publishes events) pub async fn add_message( &self, thread_id: &str, @@ -467,27 +499,29 @@ impl SessionManager { content: String, ) -> Result { use crate::session::Message; - + // Create message let message = Message::new(role, content); let message_id = message.id.clone(); - + // Save message - self.message_storage.save_message(thread_id, &message).await?; - + self.message_storage + .save_message(thread_id, &message) + .await?; + // 🔧 Update message count in session metadata let mut metadata = self.load_session(thread_id).await?; metadata.update_message_count(metadata.message_count + 1); self.update_session(&metadata).await?; - - // 🆕 发布消息添加事件 + + // 发布消息添加事件 if let Some(ref bus) = self.event_bus { let _ = bus.publish(CortexEvent::Session(SessionEvent::MessageAdded { session_id: thread_id.to_string(), message_id: message_id.clone(), })); } - + Ok(message) } } diff --git a/cortex-mem-core/src/vector_store/qdrant.rs b/cortex-mem-core/src/vector_store/qdrant.rs index 4b6d6b3..dbbbc7f 100644 --- a/cortex-mem-core/src/vector_store/qdrant.rs +++ b/cortex-mem-core/src/vector_store/qdrant.rs @@ -27,18 +27,18 @@ pub struct QdrantVectorStore { impl QdrantVectorStore { /// Create a new Qdrant vector store - /// + /// /// If `embedding_dim` is set in config, this will automatically ensure /// the collection exists (creating it if necessary). - /// - /// 🆕 If `tenant_id` is set in config, the collection name will be suffixed + /// + /// If `tenant_id` is set in config, the collection name will be suffixed /// with "_" for tenant isolation. pub async fn new(config: &QdrantConfig) -> Result { let client = Qdrant::from_url(&config.url) .build() .map_err(|e| Error::VectorStore(e))?; - // 🆕 Use tenant-aware collection name + // Use tenant-aware collection name let collection_name = config.get_collection_name(); let store = Self { @@ -56,8 +56,8 @@ impl QdrantVectorStore { } /// Create a new Qdrant vector store with auto-detected embedding dimension - /// - /// 🆕 Supports tenant isolation through config.tenant_id + /// + /// Supports tenant isolation through config.tenant_id pub async fn new_with_llm_client( config: &QdrantConfig, _llm_client: &dyn crate::llm::LLMClient, @@ -66,7 +66,7 @@ impl QdrantVectorStore { .build() .map_err(|e| Error::VectorStore(e))?; - // 🆕 Use tenant-aware collection name + // Use tenant-aware collection name let collection_name = config.get_collection_name(); let store = Self { @@ -78,12 +78,13 @@ impl QdrantVectorStore { // Auto-detect embedding dimension if not specified if store.embedding_dim.is_none() { info!("Auto-detecting embedding dimension..."); - + // Use LLMClient's embed method if available // For now, we'll require embedding_dim to be set in config return Err(Error::Config( "Embedding dimension must be specified in config when using new_with_llm_client. \ - Auto-detection from LLMClient is not yet implemented.".to_string() + Auto-detection from LLMClient is not yet implemented." + .to_string(), )); } @@ -109,7 +110,8 @@ impl QdrantVectorStore { if !collection_exists { let embedding_dim = self.embedding_dim.ok_or_else(|| { Error::Config( - "Embedding dimension not set. Use new_with_llm_client for auto-detection.".to_string() + "Embedding dimension not set. Use new_with_llm_client for auto-detection." + .to_string(), ) })?; @@ -235,29 +237,41 @@ impl QdrantVectorStore { // Store entities and topics as arrays if !memory.metadata.entities.is_empty() { - let entities_values: Vec = - memory.metadata.entities.iter() - .map(|entity| entity.to_string().into()) - .collect(); - payload.insert("entities".to_string(), qdrant_client::qdrant::Value { - kind: Some(qdrant_client::qdrant::value::Kind::ListValue( - qdrant_client::qdrant::ListValue { - values: entities_values, - })), - }); + let entities_values: Vec = memory + .metadata + .entities + .iter() + .map(|entity| entity.to_string().into()) + .collect(); + payload.insert( + "entities".to_string(), + qdrant_client::qdrant::Value { + kind: Some(qdrant_client::qdrant::value::Kind::ListValue( + qdrant_client::qdrant::ListValue { + values: entities_values, + }, + )), + }, + ); } if !memory.metadata.topics.is_empty() { - let topics_values: Vec = - memory.metadata.topics.iter() - .map(|topic| topic.to_string().into()) - .collect(); - payload.insert("topics".to_string(), qdrant_client::qdrant::Value { - kind: Some(qdrant_client::qdrant::value::Kind::ListValue( - qdrant_client::qdrant::ListValue { - values: topics_values, - })), - }); + let topics_values: Vec = memory + .metadata + .topics + .iter() + .map(|topic| topic.to_string().into()) + .collect(); + payload.insert( + "topics".to_string(), + qdrant_client::qdrant::Value { + kind: Some(qdrant_client::qdrant::value::Kind::ListValue( + qdrant_client::qdrant::ListValue { + values: topics_values, + }, + )), + }, + ); } // Custom metadata @@ -526,20 +540,28 @@ impl QdrantVectorStore { .to_string(); // Extract embedding from point vectors (VectorsOutput type from ScoredPoint) - let embedding = point.vectors.as_ref() + let embedding = point + .vectors + .as_ref() .and_then(|v| v.vectors_options.as_ref()) .and_then(|opts| match opts { vectors_output::VectorsOptions::Vector(vec) => Some(vec.data.clone()), vectors_output::VectorsOptions::Vectors(named) => { // For named vectors, try to get the default "" vector first - named.vectors.get("").cloned() + named + .vectors + .get("") + .cloned() .or_else(|| named.vectors.values().next().cloned()) .map(|v| v.data) } }) .unwrap_or_else(|| { let dim = self.embedding_dim.unwrap_or(1024); - warn!("No embedding found in point, using zero vector of dimension {}", dim); + warn!( + "No embedding found in point, using zero vector of dimension {}", + dim + ); vec![0.0; dim] }); @@ -660,20 +682,23 @@ impl QdrantVectorStore { .and_then(|v| match v { qdrant_client::qdrant::Value { kind: Some(qdrant_client::qdrant::value::Kind::ListValue(list)), - } => { - Some(list.values.iter().filter_map(|val| match val { - qdrant_client::qdrant::Value { - kind: Some(qdrant_client::qdrant::value::Kind::StringValue(s)), - } => Some(s.clone()), - _ => None, - }).collect::>()) - }, + } => Some( + list.values + .iter() + .filter_map(|val| match val { + qdrant_client::qdrant::Value { + kind: Some(qdrant_client::qdrant::value::Kind::StringValue(s)), + } => Some(s.clone()), + _ => None, + }) + .collect::>(), + ), qdrant_client::qdrant::Value { kind: Some(qdrant_client::qdrant::value::Kind::StringValue(s)), } => { // Backward compatibility: parse JSON string format serde_json::from_str(s).ok() - }, + } _ => None, }) .unwrap_or_default(), @@ -682,20 +707,23 @@ impl QdrantVectorStore { .and_then(|v| match v { qdrant_client::qdrant::Value { kind: Some(qdrant_client::qdrant::value::Kind::ListValue(list)), - } => { - Some(list.values.iter().filter_map(|val| match val { - qdrant_client::qdrant::Value { - kind: Some(qdrant_client::qdrant::value::Kind::StringValue(s)), - } => Some(s.clone()), - _ => None, - }).collect::>()) - }, + } => Some( + list.values + .iter() + .filter_map(|val| match val { + qdrant_client::qdrant::Value { + kind: Some(qdrant_client::qdrant::value::Kind::StringValue(s)), + } => Some(s.clone()), + _ => None, + }) + .collect::>(), + ), qdrant_client::qdrant::Value { kind: Some(qdrant_client::qdrant::value::Kind::StringValue(s)), } => { // Backward compatibility: parse JSON string format serde_json::from_str(s).ok() - }, + } _ => None, }) .unwrap_or_default(), @@ -944,7 +972,7 @@ impl VectorStore for QdrantVectorStore { } } } - + async fn scroll_ids(&self, filters: &Filters, limit: usize) -> Result> { let filter = self.filters_to_qdrant_filter(filters); let limit = limit as u32; @@ -964,15 +992,14 @@ impl VectorStore for QdrantVectorStore { .await .map_err(|e| Error::VectorStore(e))?; - let ids: Vec = response.result + let ids: Vec = response + .result .into_iter() .filter_map(|point| { - point.id.and_then(|id| { - match id.point_id_options { - Some(point_id::PointIdOptions::Uuid(uuid)) => Some(uuid), - Some(point_id::PointIdOptions::Num(num)) => Some(num.to_string()), - None => None, - } + point.id.and_then(|id| match id.point_id_options { + Some(point_id::PointIdOptions::Uuid(uuid)) => Some(uuid), + Some(point_id::PointIdOptions::Num(num)) => Some(num.to_string()), + None => None, }) }) .collect(); diff --git a/cortex-mem-insights/src/lib/api.ts b/cortex-mem-insights/src/lib/api.ts index c2590d8..ef5704b 100644 --- a/cortex-mem-insights/src/lib/api.ts +++ b/cortex-mem-insights/src/lib/api.ts @@ -108,7 +108,7 @@ class ApiClient { const searchRequest = { query: keyword, limit: limit, - min_score: 0.4, // 🆕 降低到0.4以支持实体查询 + min_score: 0.4, thread: scope === 'all' ? null : scope === 'user' ? 'user' : 'system' }; diff --git a/cortex-mem-mcp/src/service.rs b/cortex-mem-mcp/src/service.rs index 9d7bfc9..35c6b1f 100644 --- a/cortex-mem-mcp/src/service.rs +++ b/cortex-mem-mcp/src/service.rs @@ -158,6 +158,19 @@ pub struct IndexMemoriesResult { error_files: usize, } +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct CloseSessionArgs { + /// Thread/session ID to close + thread_id: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct CloseSessionResult { + success: bool, + thread_id: String, + message: String, +} + // ==================== MCP Tools Implementation ==================== #[tool_router] @@ -396,30 +409,43 @@ impl MemoryMcpService { ) -> std::result::Result, String> { debug!("generate_layers called with args: {:?}", params.0); - match self.operations.ensure_all_layers().await { - Ok(stats) => { - let message = if let Some(ref thread_id) = params.0.thread_id { - format!("Generated layers for session {}", thread_id) - } else { - "Generated layers for all sessions".to_string() - }; - - info!("{}: total={}, generated={}, failed={}", - message, stats.total, stats.generated, stats.failed); - - Ok(Json(GenerateLayersResult { - success: true, - message, - total: stats.total, - generated: stats.generated, - failed: stats.failed, - })) + // ✅ 根据thread_id参数选择不同的处理方式 + let (stats, message) = if let Some(ref thread_id) = params.0.thread_id { + // 只生成特定session的层级文件 + match self.operations.ensure_session_layers(thread_id).await { + Ok(stats) => { + let msg = format!("Generated layers for session {}", thread_id); + (stats, msg) + } + Err(e) => { + error!("Failed to generate layers for session {}: {}", thread_id, e); + return Err(format!("Failed to generate layers: {}", e)); + } } - Err(e) => { - error!("Failed to generate layers: {}", e); - Err(format!("Failed to generate layers: {}", e)) + } else { + // 生成所有session的层级文件 + match self.operations.ensure_all_layers().await { + Ok(stats) => { + let msg = "Generated layers for all sessions".to_string(); + (stats, msg) + } + Err(e) => { + error!("Failed to generate layers: {}", e); + return Err(format!("Failed to generate layers: {}", e)); + } } - } + }; + + info!("{}: total={}, generated={}, failed={}", + message, stats.total, stats.generated, stats.failed); + + Ok(Json(GenerateLayersResult { + success: true, + message, + total: stats.total, + generated: stats.generated, + failed: stats.failed, + })) } #[tool(description = "Index memories to vector database")] @@ -429,30 +455,71 @@ impl MemoryMcpService { ) -> std::result::Result, String> { debug!("index_memories called with args: {:?}", params.0); - match self.operations.index_all_files().await { - Ok(stats) => { - let message = if let Some(ref thread_id) = params.0.thread_id { - format!("Indexed memories for session {}", thread_id) - } else { - "Indexed all memory files".to_string() - }; - - info!("{}: total={}, indexed={}, skipped={}, errors={}", - message, stats.total_files, stats.indexed_files, - stats.skipped_files, stats.error_files); + // ✅ 根据thread_id参数选择不同的处理方式 + let (stats, message) = if let Some(ref thread_id) = params.0.thread_id { + // 只索引特定session的文件 + match self.operations.index_session_files(thread_id).await { + Ok(stats) => { + let msg = format!("Indexed memories for session {}", thread_id); + (stats, msg) + } + Err(e) => { + error!("Failed to index session {}: {}", thread_id, e); + return Err(format!("Failed to index memories: {}", e)); + } + } + } else { + // 索引所有文件 + match self.operations.index_all_files().await { + Ok(stats) => { + let msg = "Indexed all memory files".to_string(); + (stats, msg) + } + Err(e) => { + error!("Failed to index memories: {}", e); + return Err(format!("Failed to index memories: {}", e)); + } + } + }; + + info!("{}: total={}, indexed={}, skipped={}, errors={}", + message, stats.total_files, stats.indexed_files, + stats.skipped_files, stats.error_files); + + Ok(Json(IndexMemoriesResult { + success: true, + message, + total_files: stats.total_files, + indexed_files: stats.indexed_files, + skipped_files: stats.skipped_files, + error_files: stats.error_files, + })) + } + + #[tool(description = "Close a session and trigger final processing (L0/L1 generation, memory extraction, indexing)")] + async fn close_session( + &self, + params: Parameters, + ) -> std::result::Result, String> { + debug!("close_session called with args: {:?}", params.0); + + let thread_id = ¶ms.0.thread_id; + + match self.operations.close_session(thread_id).await { + Ok(_) => { + info!("Session closed successfully: {}", thread_id); - Ok(Json(IndexMemoriesResult { + Ok(Json(CloseSessionResult { success: true, - message, - total_files: stats.total_files, - indexed_files: stats.indexed_files, - skipped_files: stats.skipped_files, - error_files: stats.error_files, + thread_id: thread_id.clone(), + message: format!( + "Session closed. L0/L1 generation, memory extraction, and indexing initiated in background." + ), })) } Err(e) => { - error!("Failed to index memories: {}", e); - Err(format!("Failed to index memories: {}", e)) + error!("Failed to close session {}: {}", thread_id, e); + Err(format!("Failed to close session: {}", e)) } } } @@ -472,14 +539,23 @@ impl ServerHandler for MemoryMcpService { - get_memory: Retrieve a specific memory\n\ - delete_memory: Delete a memory\n\ - get_abstract: Get the abstract summary of a memory\n\ - - generate_layers: Generate L0/L1 layer files for memories\n\ - - index_memories: Index memories to vector database\n\ + - generate_layers: Generate L0/L1 layer files for memories (supports optional thread_id)\n\ + - index_memories: Index memories to vector database (supports optional thread_id)\n\ + - close_session: Close a session and trigger final processing\n\ \n\ URI format: cortex://{dimension}/{category}/{resource}\n\ Examples:\n\ - cortex://session/default/timeline/...\n\ - cortex://user/preferences/language.md\n\ - - cortex://agent/cases/case_001.md" + - cortex://agent/cases/case_001.md\n\ + \n\ + Session Management:\n\ + - Call close_session when conversation ends to trigger:\n\ + * L0/L1 layer generation\n\ + * Memory extraction\n\ + * Vector indexing\n\ + - Sessions are automatically created on first store_memory call\n\ + - Each session has a unique thread_id for isolation" .to_string(), ), capabilities: ServerCapabilities::builder().enable_tools().build(), diff --git a/cortex-mem-rig/src/lib.rs b/cortex-mem-rig/src/lib.rs index 7bff022..635c039 100644 --- a/cortex-mem-rig/src/lib.rs +++ b/cortex-mem-rig/src/lib.rs @@ -1,7 +1,7 @@ pub mod tools; -pub use cortex_mem_tools::MemoryOperations; pub use cortex_mem_core::llm::LLMClient; +pub use cortex_mem_tools::MemoryOperations; pub use tools::*; use std::sync::Arc; @@ -15,48 +15,48 @@ impl MemoryTools { pub fn new(operations: Arc) -> Self { Self { operations } } - + /// Get the underlying MemoryOperations pub fn operations(&self) -> &Arc { &self.operations } - + // ==================== Tiered Access Tools ==================== - + pub fn abstract_tool(&self) -> AbstractTool { AbstractTool::new(self.operations.clone()) } - + pub fn overview_tool(&self) -> OverviewTool { OverviewTool::new(self.operations.clone()) } - + pub fn read_tool(&self) -> ReadTool { ReadTool::new(self.operations.clone()) } - + // ==================== Search Tools ==================== - + pub fn search_tool(&self) -> SearchTool { SearchTool::new(self.operations.clone()) } - + pub fn find_tool(&self) -> FindTool { FindTool::new(self.operations.clone()) } - + // ==================== Filesystem Tools ==================== - + pub fn ls_tool(&self) -> LsTool { LsTool::new(self.operations.clone()) } - + pub fn explore_tool(&self) -> ExploreTool { ExploreTool::new(self.operations.clone()) } - + // ==================== Storage Tools ==================== - + pub fn store_tool(&self) -> StoreTool { StoreTool::new(self.operations.clone()) } @@ -68,7 +68,7 @@ pub fn create_memory_tools(operations: Arc) -> MemoryTools { } /// Create memory tools with full features (LLM + Vector Search) -/// +/// /// This is the primary constructor that requires all dependencies. pub async fn create_memory_tools_with_tenant_and_vector( data_dir: impl AsRef, @@ -80,7 +80,7 @@ pub async fn create_memory_tools_with_tenant_and_vector( embedding_api_key: &str, embedding_model_name: &str, embedding_dim: Option, - user_id: Option, // 🆕 添加user_id参数 + user_id: Option, ) -> Result> { let operations = MemoryOperations::new( data_dir.as_ref().to_str().unwrap(), @@ -92,7 +92,8 @@ pub async fn create_memory_tools_with_tenant_and_vector( embedding_api_key, embedding_model_name, embedding_dim, - user_id, // 🆕 传递user_id - ).await?; + user_id, + ) + .await?; Ok(MemoryTools::new(Arc::new(operations))) -} \ No newline at end of file +} diff --git a/cortex-mem-service/src/handlers/automation.rs b/cortex-mem-service/src/handlers/automation.rs index 7de9bf6..f566b59 100644 --- a/cortex-mem-service/src/handlers/automation.rs +++ b/cortex-mem-service/src/handlers/automation.rs @@ -73,50 +73,53 @@ pub async fn trigger_extraction( let entities_for_response = if req.auto_save { // 🔧 修复: 使用MemoryExtractor保存提取的记忆 use cortex_mem_core::session::extraction::MemoryExtractor; - + // 从metadata获取user_id和agent_id，如果没有则使用默认值 - let user_id = "default".to_string(); // TODO: 从请求或session metadata获取 + let user_id = "default".to_string(); // TODO: 从请求或session metadata获取 let agent_id = "default".to_string(); - + let memory_extractor = MemoryExtractor::new( llm_client.clone(), state.filesystem.clone(), user_id, agent_id, ); - + // 转换extraction_result为ExtractedMemories格式 - use cortex_mem_core::session::extraction::{ - ExtractedMemories, EntityMemory, - }; - + use cortex_mem_core::session::extraction::{EntityMemory, ExtractedMemories}; + // 先clone entities用于返回 let entities_clone = extraction_result.entities.clone(); - + let extracted_memories = ExtractedMemories { - preferences: vec![], // extraction_result不包含preferences - entities: extraction_result.entities.into_iter().map(|e| { - EntityMemory { + preferences: vec![], // extraction_result不包含preferences + entities: extraction_result + .entities + .into_iter() + .map(|e| EntityMemory { name: e.name.clone(), entity_type: e.entity_type.clone(), description: e.description.unwrap_or_else(|| e.name.clone()), context: format!("Extracted from session {}", thread_id), - } - }).collect(), - events: vec![], // extraction_result不包含events - cases: vec![], // extraction_result不包含cases + }) + .collect(), + events: vec![], // extraction_result不包含events + cases: vec![], // extraction_result不包含cases personal_info: vec![], work_history: vec![], relationships: vec![], goals: vec![], }; - + if let Err(e) = memory_extractor.save_memories(&extracted_memories).await { tracing::warn!("Failed to auto-save memories: {}", e); } else { - tracing::info!("Auto-saved {} entities to user/agent memories", extracted_memories.entities.len()); + tracing::info!( + "Auto-saved {} entities to user/agent memories", + extracted_memories.entities.len() + ); } - + entities_clone } else { extraction_result.entities @@ -150,7 +153,7 @@ pub async fn trigger_indexing( .as_ref() .ok_or_else(|| AppError::BadRequest("Embedding service not configured.".to_string()))?; - // 🆕 Create QdrantVectorStore (required for AutoIndexer) + // Create QdrantVectorStore (required for AutoIndexer) let qdrant_store = match state.create_qdrant_store().await { Ok(store) => Arc::new(store), Err(e) => { @@ -161,7 +164,7 @@ pub async fn trigger_indexing( } }; - // 🆕 Create tenant-aware filesystem + // Create tenant-aware filesystem let filesystem = if let Some(tenant_root) = state.current_tenant_root.read().await.as_ref() { Arc::new(CortexFilesystem::new( tenant_root.to_string_lossy().as_ref(), @@ -209,7 +212,7 @@ pub async fn trigger_indexing_all( .as_ref() .ok_or_else(|| AppError::BadRequest("Embedding service not configured.".to_string()))?; - // 🆕 Create QdrantVectorStore (required for AutoIndexer) + // Create QdrantVectorStore (required for AutoIndexer) let qdrant_store = match state.create_qdrant_store().await { Ok(store) => Arc::new(store), Err(e) => { @@ -220,7 +223,7 @@ pub async fn trigger_indexing_all( } }; - // 🆕 Create tenant-aware filesystem + // Create tenant-aware filesystem let filesystem = if let Some(tenant_root) = state.current_tenant_root.read().await.as_ref() { Arc::new(CortexFilesystem::new( tenant_root.to_string_lossy().as_ref(), diff --git a/cortex-mem-service/src/state.rs b/cortex-mem-service/src/state.rs index a353559..4f893b4 100644 --- a/cortex-mem-service/src/state.rs +++ b/cortex-mem-service/src/state.rs @@ -10,7 +10,7 @@ use tokio::sync::RwLock; #[derive(Clone)] pub struct AppState { #[allow(dead_code)] - pub cortex: Arc, // 🆕 统一自动索引实例 + pub cortex: Arc, pub filesystem: Arc, pub session_manager: Arc>, pub llm_client: Option>, @@ -18,13 +18,13 @@ pub struct AppState { pub vector_store: Option>, pub embedding_client: Option>, /// Vector search engine with L0/L1/L2 layered search support - /// 🆕 使用RwLock支持租户切换时重新创建 + /// 使用RwLock支持租户切换时重新创建 pub vector_engine: Arc>>>, /// Base data directory pub data_dir: PathBuf, /// Current tenant root directory (if set) pub current_tenant_root: Arc>>, - /// 🆕 Current tenant ID (for recreating tenant-specific vector store) + /// Current tenant ID (for recreating tenant-specific vector store) pub current_tenant_id: Arc>>, } @@ -66,7 +66,8 @@ impl AppState { index_on_message: true, // ✅ 实时索引（API服务需要即时搜索） index_on_close: true, index_batch_delay: 1, // 1秒批处理 - auto_generate_layers_on_startup: false, // 🆕 本地文件系统下默认关闭 + auto_generate_layers_on_startup: false, + generate_layers_every_n_messages: 5, }); // 构建Cortex Memory @@ -120,10 +121,10 @@ impl AppState { llm_client, vector_store, embedding_client, - vector_engine: Arc::new(RwLock::new(vector_engine)), // 🆕 包装在RwLock中 + vector_engine: Arc::new(RwLock::new(vector_engine)), data_dir, current_tenant_root: Arc::new(RwLock::new(None)), - current_tenant_id: Arc::new(RwLock::new(None)), // 🆕 初始化租户ID + current_tenant_id: Arc::new(RwLock::new(None)), }) } @@ -173,7 +174,7 @@ impl AppState { collection_name: config.qdrant.collection_name, embedding_dim: config.qdrant.embedding_dim, timeout_secs: config.qdrant.timeout_secs, - tenant_id: None, // 🆕 初始化时不设置租户ID（global） + tenant_id: None, }; Ok((llm_client, Some(embedding_config), Some(qdrant_config))) @@ -239,7 +240,7 @@ impl AppState { .ok() .and_then(|s| s.parse().ok()), timeout_secs: 30, - tenant_id: None, // 🆕 初始化时不设置租户ID（global） + tenant_id: None, }) } else { tracing::warn!("Qdrant not configured"); @@ -275,7 +276,7 @@ impl AppState { } /// Switch to a different tenant - /// 🆕 Recreates VectorSearchEngine with tenant-specific collection + /// Recreates VectorSearchEngine with tenant-specific collection pub async fn switch_tenant(&self, tenant_id: &str) -> anyhow::Result<()> { // Try both possible tenant locations let possible_paths = vec![ @@ -299,25 +300,25 @@ impl AppState { *current = Some(tenant_root.clone()); drop(current); - // 🆕 Update current tenant ID + // Update current tenant ID let mut current_id = self.current_tenant_id.write().await; *current_id = Some(tenant_id.to_string()); drop(current_id); tracing::info!("Switched to tenant root: {:?}", tenant_root); - // 🆕 Recreate VectorSearchEngine with tenant-specific collection + // Recreate VectorSearchEngine with tenant-specific collection if let (Some(ec), Some(llm)) = (&self.embedding_client, &self.llm_client) { let (_, _, qdrant_cfg_opt) = Self::load_configs()?; if let Some(mut qdrant_cfg) = qdrant_cfg_opt { - // 🆕 Set tenant ID in config + // Set tenant ID in config qdrant_cfg.tenant_id = Some(tenant_id.to_string()); if let Ok(qdrant_store) = cortex_mem_core::QdrantVectorStore::new(&qdrant_cfg).await { let qdrant_arc = Arc::new(qdrant_store); - // 🆕 Create tenant-specific filesystem + // Create tenant-specific filesystem let tenant_filesystem = Arc::new(CortexFilesystem::new( tenant_root.to_string_lossy().as_ref(), )); @@ -329,7 +330,7 @@ impl AppState { llm.clone(), )); - // 🆕 Update vector_engine + // Update vector_engine let mut engine = self.vector_engine.write().await; *engine = Some(new_vector_engine); @@ -345,10 +346,10 @@ impl AppState { Ok(()) } - /// 🆕 Helper method to create QdrantVectorStore for manual indexing + /// Helper method to create QdrantVectorStore for manual indexing /// This is needed because AutoIndexer requires concrete QdrantVectorStore type /// - /// 🆕 Supports tenant-specific collection + /// Supports tenant-specific collection pub async fn create_qdrant_store(&self) -> anyhow::Result { // Get current tenant ID let tenant_id = self.current_tenant_id.read().await.clone(); @@ -360,10 +361,10 @@ impl AppState { collection_name: config.qdrant.collection_name, embedding_dim: config.qdrant.embedding_dim, timeout_secs: config.qdrant.timeout_secs, - tenant_id: None, // 🆕 初始化为None + tenant_id: None, }; - // 🆕 Set tenant ID if available + // Set tenant ID if available if let Some(tid) = tenant_id { qdrant_config.tenant_id = Some(tid); } @@ -382,7 +383,7 @@ impl AppState { .ok() .and_then(|s| s.parse().ok()), timeout_secs: 30, - tenant_id, // 🆕 使用当前租户ID + tenant_id, }; cortex_mem_core::QdrantVectorStore::new(&qdrant_config) .await diff --git a/cortex-mem-tools/src/lib.rs b/cortex-mem-tools/src/lib.rs index ef0ec60..3f34b61 100644 --- a/cortex-mem-tools/src/lib.rs +++ b/cortex-mem-tools/src/lib.rs @@ -1,15 +1,15 @@ pub mod errors; -pub mod operations; -pub mod types; pub mod mcp; +pub mod operations; pub mod tools; +pub mod types; -pub use errors::{ToolsError, Result}; +pub use errors::{Result, ToolsError}; +pub use mcp::{ToolDefinition, get_mcp_tool_definition, get_mcp_tool_definitions}; pub use operations::MemoryOperations; pub use types::*; -pub use mcp::{ToolDefinition, get_mcp_tool_definitions, get_mcp_tool_definition}; pub use cortex_mem_core::automation::GenerationStats; -// 🆕 重新导出 SyncStats 以便外部使用 +// 重新导出 SyncStats 以便外部使用 pub use cortex_mem_core::automation::SyncStats; diff --git a/cortex-mem-tools/src/operations.rs b/cortex-mem-tools/src/operations.rs index e734ff4..3b8ee75 100644 --- a/cortex-mem-tools/src/operations.rs +++ b/cortex-mem-tools/src/operations.rs @@ -1,26 +1,26 @@ use crate::{errors::*, types::*}; use cortex_mem_core::{ - layers::manager::LayerManager, - llm::LLMClient, - search::VectorSearchEngine, - CortexFilesystem, + CortexFilesystem, FilesystemOperations, - SessionConfig, + SessionConfig, SessionManager, automation::{ - SyncConfig, SyncManager, AutoExtractor, AutoExtractConfig, - AutoIndexer, IndexerConfig, AutomationManager, AutomationConfig, - LayerGenerator, LayerGenerationConfig, AbstractConfig, OverviewConfig, // 🆕 添加LayerGenerator + AbstractConfig, AutoExtractConfig, AutoExtractor, AutoIndexer, AutomationConfig, + AutomationManager, IndexerConfig, LayerGenerationConfig, LayerGenerator, OverviewConfig, + SyncConfig, SyncManager, }, embedding::{EmbeddingClient, EmbeddingConfig}, - vector_store::{QdrantVectorStore, VectorStore}, // 🔧 添加VectorStore trait - events::EventBus, // 🆕 添加EventBus + events::EventBus, + layers::manager::LayerManager, + llm::LLMClient, + search::VectorSearchEngine, + vector_store::{QdrantVectorStore, VectorStore}, // 🔧 添加VectorStore trait }; use std::sync::Arc; use tokio::sync::RwLock; /// High-level memory operations with OpenViking-style tiered access -/// +/// /// All operations require: /// - LLM client for layer generation /// - Vector search engine for semantic search @@ -30,17 +30,17 @@ pub struct MemoryOperations { pub(crate) session_manager: Arc>, pub(crate) layer_manager: Arc, pub(crate) vector_engine: Arc, - pub(crate) auto_extractor: Option>, // 🆕 AutoExtractor用于退出时提取 - pub(crate) layer_generator: Option>, // 🆕 LayerGenerator用于退出时生成L0/L1 - pub(crate) auto_indexer: Option>, // 🆕 AutoIndexer用于退出时索引 - - // 🆕 保存组件引用以便退出时索引使用 + pub(crate) auto_extractor: Option>, + pub(crate) layer_generator: Option>, + pub(crate) auto_indexer: Option>, + + // 保存组件引用以便退出时索引使用 pub(crate) embedding_client: Arc, pub(crate) vector_store: Arc, pub(crate) llm_client: Arc, - - pub(crate) default_user_id: String, // 🆕 默认user_id - pub(crate) default_agent_id: String, // 🆕 默认agent_id + + pub(crate) default_user_id: String, + pub(crate) default_agent_id: String, } impl MemoryOperations { @@ -53,29 +53,29 @@ impl MemoryOperations { pub fn vector_engine(&self) -> &Arc { &self.vector_engine } - - /// 🆕 Get the session manager + + /// Get the session manager pub fn session_manager(&self) -> &Arc> { &self.session_manager } - - /// 🆕 Get the auto extractor (for manual extraction on exit) + + /// Get the auto extractor (for manual extraction on exit) pub fn auto_extractor(&self) -> Option<&Arc> { self.auto_extractor.as_ref() } - - /// 🆕 Get the layer generator (for manual layer generation on exit) + + /// Get the layer generator (for manual layer generation on exit) pub fn layer_generator(&self) -> Option<&Arc> { self.layer_generator.as_ref() } - - /// 🆕 Get the auto indexer (for manual indexing on exit) + + /// Get the auto indexer (for manual indexing on exit) pub fn auto_indexer(&self) -> Option<&Arc> { self.auto_indexer.as_ref() } /// Create from data directory with tenant isolation, LLM support, and vector search - /// + /// /// This is the primary constructor that requires all dependencies. pub async fn new( data_dir: &str, @@ -87,17 +87,17 @@ impl MemoryOperations { embedding_api_key: &str, embedding_model_name: &str, embedding_dim: Option, - user_id: Option, // 🆕 添加user_id参数 + user_id: Option, ) -> Result { let tenant_id = tenant_id.into(); let filesystem = Arc::new(CortexFilesystem::with_tenant(data_dir, &tenant_id)); filesystem.initialize().await?; - // 🆕 创建EventBus用于自动化 + // 创建EventBus用于自动化 let (event_bus, mut event_rx_main) = EventBus::new(); let config = SessionConfig::default(); - // 🆕 使用with_llm_and_events创建SessionManager + // 使用with_llm_and_events创建SessionManager let session_manager = SessionManager::with_llm_and_events( filesystem.clone(), config, @@ -116,13 +116,19 @@ impl MemoryOperations { collection_name: qdrant_collection.to_string(), embedding_dim, timeout_secs: 30, - tenant_id: Some(tenant_id.clone()), // 🆕 设置租户ID + tenant_id: Some(tenant_id.clone()), }; let vector_store = Arc::new(QdrantVectorStore::new(&qdrant_config).await?); - tracing::info!("Qdrant connected successfully, collection: {}", qdrant_config.get_collection_name()); + tracing::info!( + "Qdrant connected successfully, collection: {}", + qdrant_config.get_collection_name() + ); // Initialize Embedding client - tracing::info!("Initializing Embedding client with model: {}", embedding_model_name); + tracing::info!( + "Initializing Embedding client with model: {}", + embedding_model_name + ); let embedding_config = EmbeddingConfig { api_base_url: embedding_api_base_url.to_string(), api_key: embedding_api_key.to_string(), @@ -142,13 +148,13 @@ impl MemoryOperations { )); tracing::info!("Vector search engine created with LLM support for query rewriting"); - // 🆕 使用传入的user_id，如果没有则使用tenant_id + // 使用传入的user_id，如果没有则使用tenant_id let actual_user_id = user_id.unwrap_or_else(|| tenant_id.clone()); - + // 🔧 创建AutoExtractor(简化配置，移除了save_user_memories和save_agent_memories) let auto_extract_config = AutoExtractConfig { min_message_count: 5, - extract_on_close: true, // 🔧 显式设置为true，确保会话关闭时自动提取记忆 + extract_on_close: true, // 🔧 显式设置为true，确保会话关闭时自动提取记忆 }; let auto_extractor = Arc::new(AutoExtractor::with_user_id( filesystem.clone(), @@ -156,8 +162,8 @@ impl MemoryOperations { auto_extract_config, &actual_user_id, )); - - // 🆕 创建AutoIndexer用于实时索引 + + // 创建AutoIndexer用于实时索引 let indexer_config = IndexerConfig { auto_index: true, batch_size: 10, @@ -169,19 +175,19 @@ impl MemoryOperations { vector_store.clone(), indexer_config, )); - - // 🆕 创建AutomationManager + + // 创建AutomationManager let automation_config = AutomationConfig { auto_index: true, - auto_extract: false, // Extract由单独的监听器处理 - index_on_message: true, // ✅ 消息时自动索引L2 - index_on_close: true, // ✅ Session关闭时生成L0/L1并索引 + auto_extract: false, // Extract由单独的监听器处理 + index_on_message: true, // ✅ 消息时自动索引L2 + index_on_close: true, // ✅ Session关闭时生成L0/L1并索引 index_batch_delay: 1, - auto_generate_layers_on_startup: false, // 🆕 启动时不生成（避免阻塞） - generate_layers_every_n_messages: 5, // 🆕 每5条消息生成一次L0/L1 + auto_generate_layers_on_startup: false, // 启动时不生成（避免阻塞） + generate_layers_every_n_messages: 5, // 每5条消息生成一次L0/L1 }; - - // 🆕 创建LayerGenerator（用于退出时手动生成） + + // 创建LayerGenerator（用于退出时手动生成） let layer_gen_config = LayerGenerationConfig { batch_size: 10, delay_ms: 1000, @@ -201,18 +207,18 @@ impl MemoryOperations { llm_client.clone(), layer_gen_config, )); - + let automation_manager = AutomationManager::new( auto_indexer.clone(), - None, // extractor由单独的监听器处理 + None, // extractor由单独的监听器处理 automation_config, ) - .with_layer_generator(layer_generator.clone()); // 🆕 设置LayerGenerator - - // 🆕 创建事件转发器（将主EventBus的事件转发给两个监听器） + .with_layer_generator(layer_generator.clone()); // 设置LayerGenerator + + // 创建事件转发器（将主EventBus的事件转发给两个监听器） let (tx_automation, rx_automation) = tokio::sync::mpsc::unbounded_channel(); let (tx_extractor, rx_extractor) = tokio::sync::mpsc::unbounded_channel(); - + tokio::spawn(async move { while let Some(event) = event_rx_main.recv().await { // 转发给AutomationManager @@ -221,21 +227,27 @@ impl MemoryOperations { let _ = tx_extractor.send(event); } }); - - // 🆕 启动AutomationManager监听事件并自动索引 + + // 启动AutomationManager监听事件并自动索引 let tenant_id_for_automation = tenant_id.clone(); tokio::spawn(async move { - tracing::info!("Starting AutomationManager for tenant {}", tenant_id_for_automation); + tracing::info!( + "Starting AutomationManager for tenant {}", + tenant_id_for_automation + ); if let Err(e) = automation_manager.start(rx_automation).await { tracing::error!("AutomationManager stopped with error: {}", e); } }); - - // 🆕 启动后台监听器处理SessionClosed事件 + + // 启动后台监听器处理SessionClosed事件 let extractor_clone = auto_extractor.clone(); let tenant_id_clone = tenant_id.clone(); tokio::spawn(async move { - tracing::info!("Starting AutoExtractor event listener for tenant {}", tenant_id_clone); + tracing::info!( + "Starting AutoExtractor event listener for tenant {}", + tenant_id_clone + ); let mut rx = rx_extractor; while let Some(event) = rx.recv().await { if let cortex_mem_core::CortexEvent::Session(session_event) = event { @@ -259,7 +271,7 @@ impl MemoryOperations { } } } - _ => {} // 忽略其他事件 + _ => {} // 忽略其他事件 } } } @@ -297,23 +309,27 @@ impl MemoryOperations { session_manager, layer_manager, vector_engine, - auto_extractor: Some(auto_extractor), // 🆕 - layer_generator: Some(layer_generator), // 🆕 保存LayerGenerator用于退出时生成 - auto_indexer: Some(auto_indexer), // 🆕 保存AutoIndexer用于退出时索引 - - // 🆕 保存组件引用以便退出时索引使用 + auto_extractor: Some(auto_extractor), + layer_generator: Some(layer_generator), // 保存LayerGenerator用于退出时生成 + auto_indexer: Some(auto_indexer), // 保存AutoIndexer用于退出时索引 + + // 保存组件引用以便退出时索引使用 embedding_client, vector_store, llm_client, - - default_user_id: actual_user_id, // 🆕 存储默认user_id - default_agent_id: tenant_id.clone(), // 🆕 使用tenant_id作为默认agent_id + + default_user_id: actual_user_id, + default_agent_id: tenant_id.clone(), }) } /// Add a message to a session pub async fn add_message(&self, thread_id: &str, role: &str, content: &str) -> Result { - let thread_id = if thread_id.is_empty() { "default" } else { thread_id }; + let thread_id = if thread_id.is_empty() { + "default" + } else { + thread_id + }; let sm = self.session_manager.read().await; @@ -325,17 +341,18 @@ impl MemoryOperations { thread_id, Some(self.default_user_id.clone()), Some(self.default_agent_id.clone()), - ).await?; + ) + .await?; drop(sm); } else { // 🔧 Session存在，检查并更新user_id/agent_id（兼容旧session） if let Ok(metadata) = sm.load_session(thread_id).await { let needs_update = metadata.user_id.is_none() || metadata.agent_id.is_none(); - + if needs_update { drop(sm); let sm = self.session_manager.write().await; - + // 重新加载并更新 if let Ok(mut metadata) = sm.load_session(thread_id).await { if metadata.user_id.is_none() { @@ -362,8 +379,10 @@ impl MemoryOperations { "system" => cortex_mem_core::MessageRole::System, _ => cortex_mem_core::MessageRole::User, }; - - let message = sm.add_message(thread_id, message_role, content.to_string()).await?; + + let message = sm + .add_message(thread_id, message_role, content.to_string()) + .await?; let message_uri = format!( "cortex://session/{}/timeline/{}/{}/{}_{}.md", thread_id, @@ -373,7 +392,11 @@ impl MemoryOperations { &message.id[..8] ); - tracing::info!("Added message to session {}, URI: {}", thread_id, message_uri); + tracing::info!( + "Added message to session {}, URI: {}", + thread_id, + message_uri + ); Ok(message_uri) } @@ -385,7 +408,13 @@ impl MemoryOperations { for entry in entries { if entry.is_directory { let thread_id = entry.name; - if let Ok(metadata) = self.session_manager.read().await.load_session(&thread_id).await { + if let Ok(metadata) = self + .session_manager + .read() + .await + .load_session(&thread_id) + .await + { let status_str = match metadata.status { cortex_mem_core::session::manager::SessionStatus::Active => "active", cortex_mem_core::session::manager::SessionStatus::Closed => "closed", @@ -451,18 +480,25 @@ impl MemoryOperations { pub async fn delete(&self, uri: &str) -> Result<()> { // First delete from vector database // We need to delete all 3 layers: L0, L1, L2 - let l0_id = cortex_mem_core::uri_to_vector_id(uri, cortex_mem_core::ContextLayer::L0Abstract); - let l1_id = cortex_mem_core::uri_to_vector_id(uri, cortex_mem_core::ContextLayer::L1Overview); + let l0_id = + cortex_mem_core::uri_to_vector_id(uri, cortex_mem_core::ContextLayer::L0Abstract); + let l1_id = + cortex_mem_core::uri_to_vector_id(uri, cortex_mem_core::ContextLayer::L1Overview); let l2_id = cortex_mem_core::uri_to_vector_id(uri, cortex_mem_core::ContextLayer::L2Detail); - + // Delete from vector store (ignore errors as vectors might not exist) let _ = self.vector_store.delete(&l0_id).await; let _ = self.vector_store.delete(&l1_id).await; let _ = self.vector_store.delete(&l2_id).await; - - tracing::info!("Deleted vectors for URI: {} (L0: {}, L1: {}, L2: {})", - uri, l0_id, l1_id, l2_id); - + + tracing::info!( + "Deleted vectors for URI: {} (L0: {}, L1: {}, L2: {})", + uri, + l0_id, + l1_id, + l2_id + ); + // Then delete from filesystem self.filesystem.delete(uri).await?; tracing::info!("Deleted file: {}", uri); @@ -471,12 +507,16 @@ impl MemoryOperations { /// Check if file/directory exists pub async fn exists(&self, uri: &str) -> Result { - let exists = self.filesystem.exists(uri).await.map_err(ToolsError::Core)?; + let exists = self + .filesystem + .exists(uri) + .await + .map_err(ToolsError::Core)?; Ok(exists) } - - /// 🆕 生成所有缺失的 L0/L1 层级文件（用于退出时调用） - /// + + /// 生成所有缺失的 L0/L1 层级文件（用于退出时调用） + /// /// 这个方法扫描所有目录，找出缺失 .abstract.md 或 .overview.md 的目录， /// 并批量生成它们。适合在应用退出时调用。 pub async fn ensure_all_layers(&self) -> Result { @@ -486,7 +526,9 @@ impl MemoryOperations { Ok(stats) => { tracing::info!( "✅ L0/L1 层级生成完成: 总计 {}, 成功 {}, 失败 {}", - stats.total, stats.generated, stats.failed + stats.total, + stats.generated, + stats.failed ); Ok(stats) } @@ -500,25 +542,60 @@ impl MemoryOperations { Ok(cortex_mem_core::automation::GenerationStats::default()) } } - - /// 🆕 索引所有文件到向量数据库（用于退出时调用） - /// + + /// 为特定session生成 L0/L1 层级文件 + /// # Arguments + /// * `session_id` - 会话ID + /// + /// # Returns + /// 返回生成统计信息 + pub async fn ensure_session_layers( + &self, + session_id: &str, + ) -> Result { + if let Some(ref generator) = self.layer_generator { + let timeline_uri = format!("cortex://session/{}/timeline", session_id); + tracing::info!("🔍 为会话 {} 生成 L0/L1 层级文件", session_id); + + match generator.ensure_timeline_layers(&timeline_uri).await { + Ok(stats) => { + tracing::info!( + "✅ 会话 {} L0/L1 层级生成完成: 总计 {}, 成功 {}, 失败 {}", + session_id, + stats.total, + stats.generated, + stats.failed + ); + Ok(stats) + } + Err(e) => { + tracing::error!("❌ 会话 {} L0/L1 层级生成失败: {}", session_id, e); + Err(e.into()) + } + } + } else { + tracing::warn!("⚠️ LayerGenerator 未配置，跳过层级生成"); + Ok(cortex_mem_core::automation::GenerationStats::default()) + } + } + + /// 索引所有文件到向量数据库（用于退出时调用） /// 这个方法扫描所有文件，包括新生成的 .abstract.md 和 .overview.md， /// 并将它们索引到向量数据库中。适合在应用退出时调用。 pub async fn index_all_files(&self) -> Result { tracing::info!("📊 开始索引所有文件到向量数据库..."); - - use cortex_mem_core::automation::{SyncManager, SyncConfig}; - + + use cortex_mem_core::automation::{SyncConfig, SyncManager}; + // 创建 SyncManager let sync_manager = SyncManager::new( self.filesystem.clone(), self.embedding_client.clone(), self.vector_store.clone(), - self.llm_client.clone(), // 不需要 Option + self.llm_client.clone(), // 不需要 Option SyncConfig::default(), ); - + match sync_manager.sync_all().await { Ok(stats) => { tracing::info!( @@ -536,4 +613,50 @@ impl MemoryOperations { } } } -} \ No newline at end of file + + /// 为特定session索引文件到向量数据库 + /// + /// # Arguments + /// * `session_id` - 会话ID + /// + /// # Returns + /// 返回索引统计信息 + pub async fn index_session_files( + &self, + session_id: &str, + ) -> Result { + tracing::info!("📊 开始为会话 {} 索引文件到向量数据库...", session_id); + + use cortex_mem_core::automation::{SyncConfig, SyncManager}; + + // 创建 SyncManager + let sync_manager = SyncManager::new( + self.filesystem.clone(), + self.embedding_client.clone(), + self.vector_store.clone(), + self.llm_client.clone(), + SyncConfig::default(), + ); + + // 限定扫描范围到特定session + let session_uri = format!("cortex://session/{}", session_id); + + match sync_manager.sync_specific_path(&session_uri).await { + Ok(stats) => { + tracing::info!( + "✅ 会话 {} 索引完成: 总计 {} 个文件, {} 个已索引, {} 个跳过, {} 个错误", + session_id, + stats.total_files, + stats.indexed_files, + stats.skipped_files, + stats.error_files + ); + Ok(stats) + } + Err(e) => { + tracing::error!("❌ 会话 {} 索引失败: {}", session_id, e); + Err(e.into()) + } + } + } +} diff --git a/cortex-mem-tools/src/tools/storage.rs b/cortex-mem-tools/src/tools/storage.rs index a50d7f8..adc27db 100644 --- a/cortex-mem-tools/src/tools/storage.rs +++ b/cortex-mem-tools/src/tools/storage.rs @@ -1,9 +1,9 @@ // Storage Tools - Store content with automatic layer generation -use crate::{Result, types::*, MemoryOperations}; -use cortex_mem_core::{MessageRole, FilesystemOperations}; -use std::collections::HashMap; +use crate::{MemoryOperations, Result, types::*}; use chrono::Utc; +use cortex_mem_core::{FilesystemOperations, MessageRole}; +use std::collections::HashMap; impl MemoryOperations { /// Store content with automatic L0/L1 layer generation @@ -13,7 +13,7 @@ impl MemoryOperations { "user" | "session" | "agent" => args.scope.as_str(), _ => "session", // Default to session }; - + // Build URI based on scope let uri = match scope { "user" => { @@ -25,14 +25,29 @@ impl MemoryOperations { let filename = format!( "{}_{}.md", now.format("%H_%M_%S"), - uuid::Uuid::new_v4().to_string().split('-').next().unwrap_or("unknown") + uuid::Uuid::new_v4() + .to_string() + .split('-') + .next() + .unwrap_or("unknown") ); - format!("cortex://user/{}/memories/{}/{}/{}", user_id, year_month, day, filename) - }, + format!( + "cortex://user/{}/memories/{}/{}/{}", + user_id, year_month, day, filename + ) + } "agent" => { // cortex://agent/{agent_id}/memories/YYYY-MM/DD/HH_MM_SS_id.md - let agent_id = args.agent_id.as_deref() - .or_else(|| if args.thread_id.is_empty() { None } else { Some(&args.thread_id) }) + let agent_id = args + .agent_id + .as_deref() + .or_else(|| { + if args.thread_id.is_empty() { + None + } else { + Some(&args.thread_id) + } + }) .unwrap_or("default"); let now = Utc::now(); let year_month = now.format("%Y-%m").to_string(); @@ -40,10 +55,17 @@ impl MemoryOperations { let filename = format!( "{}_{}.md", now.format("%H_%M_%S"), - uuid::Uuid::new_v4().to_string().split('-').next().unwrap_or("unknown") + uuid::Uuid::new_v4() + .to_string() + .split('-') + .next() + .unwrap_or("unknown") ); - format!("cortex://agent/{}/memories/{}/{}/{}", agent_id, year_month, day, filename) - }, + format!( + "cortex://agent/{}/memories/{}/{}/{}", + agent_id, year_month, day, filename + ) + } "session" => { // cortex://session/{thread_id}/timeline/YYYY-MM/DD/HH_MM_SS_id.md let thread_id = if args.thread_id.is_empty() { @@ -51,47 +73,59 @@ impl MemoryOperations { } else { args.thread_id.clone() }; - + // 🔧 Fix: Release lock immediately after operations let message = { let sm = self.session_manager.write().await; - + // 🔧 Ensure session exists with user_id and agent_id if !sm.session_exists(&thread_id).await? { // 使用create_session_with_ids传入user_id和agent_id sm.create_session_with_ids( &thread_id, - args.user_id.clone().or_else(|| Some(self.default_user_id.clone())), - args.agent_id.clone().or_else(|| Some(self.default_agent_id.clone())), - ).await?; + args.user_id + .clone() + .or_else(|| Some(self.default_user_id.clone())), + args.agent_id + .clone() + .or_else(|| Some(self.default_agent_id.clone())), + ) + .await?; } else { // 🔧 如果session已存在但缺少user_id/agent_id，更新它 if let Ok(mut metadata) = sm.load_session(&thread_id).await { let mut needs_update = false; - + if metadata.user_id.is_none() { - metadata.user_id = args.user_id.clone().or_else(|| Some(self.default_user_id.clone())); + metadata.user_id = args + .user_id + .clone() + .or_else(|| Some(self.default_user_id.clone())); needs_update = true; } if metadata.agent_id.is_none() { - metadata.agent_id = args.agent_id.clone().or_else(|| Some(self.default_agent_id.clone())); + metadata.agent_id = args + .agent_id + .clone() + .or_else(|| Some(self.default_agent_id.clone())); needs_update = true; } - + if needs_update { let _ = sm.update_session(&metadata).await; } } } - - // 🆕 使用add_message()发布事件，而不是直接调用save_message() + + // 使用add_message()发布事件，而不是直接调用save_message() sm.add_message( &thread_id, - MessageRole::User, // 默认使用User角色 - args.content.clone() - ).await? + MessageRole::User, // 默认使用User角色 + args.content.clone(), + ) + .await? }; // Lock is released here - + // 返回消息URI let year_month = message.timestamp.format("%Y-%m").to_string(); let day = message.timestamp.format("%d").to_string(); @@ -104,26 +138,30 @@ impl MemoryOperations { "cortex://session/{}/timeline/{}/{}/{}", thread_id, year_month, day, filename ) - }, + } _ => unreachable!(), }; - + // For user and agent scope, directly write to filesystem if scope == "user" || scope == "agent" { self.filesystem.write(&uri, &args.content).await?; } - + // 🔧 Auto-generate layers if requested (ONLY for user and agent scope) // Session scope: skip per-message layer generation to avoid overwriting // Session-level layers will be generated when the session closes let layers_generated = HashMap::new(); if args.auto_generate_layers.unwrap_or(true) && scope != "session" { // Use layer_manager to generate all layers - if let Err(e) = self.layer_manager.generate_all_layers(&uri, &args.content).await { + if let Err(e) = self + .layer_manager + .generate_all_layers(&uri, &args.content) + .await + { tracing::warn!("Failed to generate layers for {}: {}", uri, e); } } - + Ok(StoreResponse { uri, layers_generated, diff --git a/examples/cortex-mem-tars/src/agent.rs b/examples/cortex-mem-tars/src/agent.rs index 9cd0540..3ef19f0 100644 --- a/examples/cortex-mem-tars/src/agent.rs +++ b/examples/cortex-mem-tars/src/agent.rs @@ -46,7 +46,7 @@ impl ChatMessage { pub fn assistant(content: impl Into) -> Self { Self::new(MessageRole::Assistant, content.into()) } - + pub fn system(content: impl Into) -> Self { Self::new(MessageRole::System, content.into()) } @@ -60,7 +60,7 @@ pub async fn create_memory_agent( user_info: Option<&str>, bot_system_prompt: Option<&str>, agent_id: &str, - user_id: &str, // 🔧 移除下划线前缀 + user_id: &str, // 🔧 移除下划线前缀 ) -> Result<(RigAgent, Arc), Box> { // 创建 cortex LLMClient 用于 L0/L1 生成 let llm_config = cortex_mem_core::llm::LLMConfig { @@ -75,7 +75,11 @@ pub async fn create_memory_agent( // 使用向量搜索版本（唯一支持的版本） tracing::info!("🔍 使用向量搜索功能"); - tracing::info!("Embedding 配置: model={}, dim={:?}", config.embedding.model_name, config.qdrant.embedding_dim); + tracing::info!( + "Embedding 配置: model={}, dim={:?}", + config.embedding.model_name, + config.qdrant.embedding_dim + ); let memory_tools = create_memory_tools_with_tenant_and_vector( data_dir, agent_id, @@ -86,7 +90,7 @@ pub async fn create_memory_agent( &config.embedding.api_key, &config.embedding.model_name, config.qdrant.embedding_dim, - Some(user_id.to_string()), // 🆕 传递真实的user_id + Some(user_id.to_string()), ) .await?; @@ -288,10 +292,10 @@ pub async fn create_memory_agent( // 构建带有 OpenViking 风格记忆工具的 agent use rig::client::CompletionClient; let completion_model = llm_client - .completions_api() // Use completions API to get CompletionModel + .completions_api() // Use completions API to get CompletionModel .agent(&config.llm.model_efficient) .preamble(&system_prompt) - .default_max_turns(30) // 🔧 设置默认max_turns为30，避免频繁触发MaxTurnError + .default_max_turns(30) // 🔧 设置默认max_turns为30，避免频繁触发MaxTurnError // 搜索工具（最常用） .tool(memory_tools.search_tool()) .tool(memory_tools.find_tool()) @@ -307,8 +311,8 @@ pub async fn create_memory_agent( } /// 从记忆中提取用户基本信息 -/// 🆕 提取用户基本信息用于初始化 Agent 上下文 -/// +/// 提取用户基本信息用于初始化 Agent 上下文 +/// /// 优化策略： /// - 优先读取目录的 .overview.md（L1 层级） /// - 如果没有 overview，回退到读取个别文件 @@ -321,7 +325,7 @@ pub async fn extract_user_basic_info( use cortex_mem_core::FilesystemOperations; tracing::info!("Loading user memories (L1 overviews) for user: {}", user_id); - + let mut context = String::new(); context.push_str("## 用户记忆\n\n"); let mut has_content = false; @@ -332,12 +336,12 @@ pub async fn extract_user_basic_info( ("work_history", "工作经历"), ("preferences", "偏好习惯"), ]; - + for (category, title) in core_categories { let category_uri = format!("cortex://user/{}/{}", user_id, category); let overview_uri = format!("{}/.overview.md", category_uri); - - // 🆕 优先读取 .overview.md（L1 层级） + + // 优先读取 .overview.md（L1 层级） if let Ok(overview_content) = operations.filesystem().read(&overview_uri).await { context.push_str(&format!("### {}\n", title)); // 移除 **Added** 时间戳 @@ -375,12 +379,12 @@ pub async fn extract_user_basic_info( ("entities", "相关实体"), ("events", "重要事件"), ]; - + for (category, title) in secondary_categories { let category_uri = format!("cortex://user/{}/{}", user_id, category); let overview_uri = format!("{}/.overview.md", category_uri); - - // 🆕 仅读取 .overview.md，不回退到详细文件 + + // 仅读取 .overview.md，不回退到详细文件 if let Ok(overview_content) = operations.filesystem().read(&overview_uri).await { context.push_str(&format!("### {}\n", title)); let clean_content = strip_metadata(&overview_content); @@ -391,10 +395,10 @@ pub async fn extract_user_basic_info( } } - // 🆕 读取 Agent 经验案例（仅 overview） + // 读取 Agent 经验案例（仅 overview） let cases_uri = format!("cortex://agent/{}/cases", _agent_id); let cases_overview_uri = format!("{}/.overview.md", cases_uri); - + if let Ok(overview_content) = operations.filesystem().read(&cases_overview_uri).await { context.push_str("### Agent经验案例\n"); let clean_content = strip_metadata(&overview_content); @@ -416,16 +420,19 @@ pub async fn extract_user_basic_info( /// 移除 **Added** 时间戳等元数据 fn strip_metadata(content: &str) -> String { let mut lines: Vec<&str> = content.lines().collect(); - + // 移除末尾的 **Added** 行 while let Some(last_line) = lines.last() { - if last_line.trim().is_empty() || last_line.contains("**Added**") || last_line.starts_with("---") { + if last_line.trim().is_empty() + || last_line.contains("**Added**") + || last_line.starts_with("---") + { lines.pop(); } else { break; } } - + lines.join("\n").trim().to_string() } @@ -433,15 +440,15 @@ fn strip_metadata(content: &str) -> String { fn extract_markdown_summary(content: &str) -> String { let mut summary = String::new(); let mut in_content = false; - + for line in content.lines() { let trimmed = line.trim(); - + // 跳过空行 if trimmed.is_empty() { continue; } - + // 提取标题（去掉#号） if trimmed.starts_with('#') { let title = trimmed.trim_start_matches('#').trim(); @@ -460,7 +467,7 @@ fn extract_markdown_summary(content: &str) -> String { summary.push_str(": "); } summary.push_str(desc); - break; // 找到描述后就返回 + break; // 找到描述后就返回 } } // 提取普通内容行（不是markdown格式的） @@ -476,13 +483,13 @@ fn extract_markdown_summary(content: &str) -> String { } } } - + // 限制长度 if summary.len() > 200 { summary.truncate(197); summary.push_str("..."); } - + summary } @@ -571,39 +578,37 @@ impl AgentChatHandler { let mut stream = agent .stream_chat(prompt_message, chat_history) - .multi_turn(30) // 🔧 从20增加到30，减少触发MaxTurnError的可能性 + .multi_turn(30) // 🔧 从20增加到30，减少触发MaxTurnError的可能性 .await; while let Some(item) = stream.next().await { match item { - Ok(stream_item) => { - match stream_item { - MultiTurnStreamItem::StreamAssistantItem(content) => { - use rig::streaming::StreamedAssistantContent; - match content { - StreamedAssistantContent::Text(text_content) => { - let text = &text_content.text; - full_response.push_str(text); - if tx.send(text.clone()).await.is_err() { - break; - } + Ok(stream_item) => match stream_item { + MultiTurnStreamItem::StreamAssistantItem(content) => { + use rig::streaming::StreamedAssistantContent; + match content { + StreamedAssistantContent::Text(text_content) => { + let text = &text_content.text; + full_response.push_str(text); + if tx.send(text.clone()).await.is_err() { + break; } - StreamedAssistantContent::ToolCall { .. } => { - log::debug!("调用工具中..."); - } - _ => {} } - } - MultiTurnStreamItem::FinalResponse(final_resp) => { - full_response = final_resp.response().to_string(); - let _ = tx.send(full_response.clone()).await; - break; - } - _ => { - log::debug!("收到其他类型的流式项目"); + StreamedAssistantContent::ToolCall { .. } => { + log::debug!("调用工具中..."); + } + _ => {} } } - } + MultiTurnStreamItem::FinalResponse(final_resp) => { + full_response = final_resp.response().to_string(); + let _ = tx.send(full_response.clone()).await; + break; + } + _ => { + log::debug!("收到其他类型的流式项目"); + } + }, Err(e) => { log::error!("流式处理错误: {:?}", e); let error_msg = format!("[错误: {}]", e); @@ -622,8 +627,8 @@ impl AgentChatHandler { scope: "session".to_string(), metadata: None, auto_generate_layers: Some(true), - user_id: Some("tars_user".to_string()), // 🔧 传递user_id - agent_id: None, // 🔧 agent_id由tenant_id决定，这里不传 + user_id: Some("tars_user".to_string()), // 🔧 传递user_id + agent_id: None, // 🔧 agent_id由tenant_id决定，这里不传 }; if let Err(e) = ops.store(user_store).await { tracing::warn!("Failed to save user message: {}", e); @@ -637,8 +642,8 @@ impl AgentChatHandler { scope: "session".to_string(), metadata: None, auto_generate_layers: Some(true), - user_id: Some("tars_user".to_string()), // 🔧 传递user_id - agent_id: None, // 🔧 agent_id由tenant_id决定，这里不传 + user_id: Some("tars_user".to_string()), // 🔧 传递user_id + agent_id: None, // 🔧 agent_id由tenant_id决定，这里不传 }; if let Err(e) = ops.store(assistant_store).await { tracing::warn!("Failed to save assistant message: {}", e); @@ -664,4 +669,4 @@ impl AgentChatHandler { Ok(response) } -} \ No newline at end of file +} diff --git a/examples/cortex-mem-tars/src/app.rs b/examples/cortex-mem-tars/src/app.rs index 535ef50..79ef0d6 100644 --- a/examples/cortex-mem-tars/src/app.rs +++ b/examples/cortex-mem-tars/src/app.rs @@ -1302,28 +1302,32 @@ impl App { log::warn!("⚠️ 会话关闭失败: {}", e); } } - - // 🆕 退出时生成所有缺失的 L0/L1 层级文件 + + // 退出时生成所有缺失的 L0/L1 层级文件 log::info!("📑 开始生成缺失的 L0/L1 层级文件..."); match tenant_ops.ensure_all_layers().await { Ok(stats) => { log::info!( "✅ 层级文件生成完成: 总计 {}, 成功 {}, 失败 {}", - stats.total, stats.generated, stats.failed + stats.total, + stats.generated, + stats.failed ); } Err(e) => { log::warn!("⚠️ 层级文件生成失败: {}", e); } } - - // 🆕 退出时索引所有文件到向量数据库 + + // 退出时索引所有文件到向量数据库 log::info!("📊 开始索引所有文件到向量数据库..."); match tenant_ops.index_all_files().await { Ok(stats) => { log::info!( "✅ 索引完成: 总计 {} 个文件, {} 个已索引, {} 个跳过", - stats.total_files, stats.indexed_files, stats.skipped_files + stats.total_files, + stats.indexed_files, + stats.skipped_files ); } Err(e) => { diff --git a/examples/cortex-mem-tars/src/audio_transcription.rs b/examples/cortex-mem-tars/src/audio_transcription.rs index 0c5ecb6..11ab4ba 100644 --- a/examples/cortex-mem-tars/src/audio_transcription.rs +++ b/examples/cortex-mem-tars/src/audio_transcription.rs @@ -347,13 +347,13 @@ pub fn is_meaningful_text(text: &str, audio_volume: f32) -> bool { return false; } - // 6. 🆕 检查是否为重复字符模式（如 "啊啊啊啊", "嗯嗯嗯"） + // 6. 检查是否为重复字符模式（如 "啊啊啊啊", "嗯嗯嗯"） if is_repetitive_pattern(text) { log::debug!("检测到重复字符模式: {}", text); return false; } - // 7. 🆕 检查是否为疑似噪音误识别（单个音节重复或无意义组合） + // 7. 检查是否为疑似噪音误识别（单个音节重复或无意义组合） if is_likely_noise_misrecognition(text) { log::debug!("检测到疑似噪音误识别: {}", text); return false; From ba9d34ae5dc68f2abf0098ef79df9e06856b3f3a Mon Sep 17 00:00:00 2001 From: Sopaco Date: Thu, 26 Feb 2026 21:52:56 +0800 Subject: [PATCH 4/5] Adjust default search scoring thresholds Increase default threshold in the vector engine (SearchOptions) and the service search handler to 0.6. Lower the API client's min_score to 0.1 to allow broader frontend results. --- cortex-mem-core/src/search/vector_engine.rs | 2 +- cortex-mem-insights/src/lib/api.ts | 2 +- cortex-mem-service/src/handlers/search.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cortex-mem-core/src/search/vector_engine.rs b/cortex-mem-core/src/search/vector_engine.rs index d51f6d0..7040cf5 100644 --- a/cortex-mem-core/src/search/vector_engine.rs +++ b/cortex-mem-core/src/search/vector_engine.rs @@ -26,7 +26,7 @@ impl Default for SearchOptions { fn default() -> Self { Self { limit: 10, - threshold: 0.5, + threshold: 0.6, root_uri: None, recursive: true, } diff --git a/cortex-mem-insights/src/lib/api.ts b/cortex-mem-insights/src/lib/api.ts index ef5704b..2b8c4e3 100644 --- a/cortex-mem-insights/src/lib/api.ts +++ b/cortex-mem-insights/src/lib/api.ts @@ -108,7 +108,7 @@ class ApiClient { const searchRequest = { query: keyword, limit: limit, - min_score: 0.4, + min_score: 0.1, thread: scope === 'all' ? null : scope === 'user' ? 'user' : 'system' }; diff --git a/cortex-mem-service/src/handlers/search.rs b/cortex-mem-service/src/handlers/search.rs index 919761b..b6d525a 100644 --- a/cortex-mem-service/src/handlers/search.rs +++ b/cortex-mem-service/src/handlers/search.rs @@ -13,7 +13,7 @@ pub async fn search( Json(req): Json, ) -> Result>>> { let limit = req.limit.unwrap_or(10); - let min_score = req.min_score.unwrap_or(0.5); + let min_score = req.min_score.unwrap_or(0.6); let results = search_layered(&state, &req.query, req.thread.as_deref(), limit, min_score).await?; From 08f7b612082c4e16acd1adbf98ab907e1e3b44f6 Mon Sep 17 00:00:00 2001 From: Sopaco Date: Thu, 26 Feb 2026 22:04:23 +0800 Subject: [PATCH 5/5] Bump package to 2.x --- cortex-mem-insights/package.json | 54 ++++++++++---------- cortex-mem-service/Cargo.toml | 2 +- cortex-mem-service/src/main.rs | 2 +- examples/cortex-mem-tars/config.example.toml | 6 +-- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cortex-mem-insights/package.json b/cortex-mem-insights/package.json index 66a5421..50b77c9 100644 --- a/cortex-mem-insights/package.json +++ b/cortex-mem-insights/package.json @@ -1,29 +1,29 @@ { - "name": "cortex-mem-insights", - "private": true, - "version": "0.1.0", - "type": "module", - "scripts": { - "dev": "vite", - "build": "vite build", - "preview": "vite preview", - "serve": "bun server.ts", - "compile": "bun build --compile --minify --bytecode --sourcemap ./server.ts --outfile ./dist/cortex-mem-insights", - "compile:prod": "bun build --compile --minify --bytecode --sourcemap --define VERSION='\"0.1.0\"' --define BUILD_TIME='\"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'\"' ./server.ts --outfile ./dist/cortex-mem-insights", - "compile:mac": "bun build --compile --minify --bytecode --target=bun-darwin-arm64 --define VERSION='\"0.1.0\"' ./server.ts --outfile ./dist/cortex-mem-insights-mac-arm64", - "compile:mac-x64": "bun build --compile --minify --bytecode --target=bun-darwin-x64 --define VERSION='\"0.1.0\"' ./server.ts --outfile ./dist/cortex-mem-insights-mac-x64", - "compile:linux": "bun build --compile --minify --bytecode --target=bun-linux-x64 --define VERSION='\"0.1.0\"' ./server.ts --outfile ./dist/cortex-mem-insights-linux", - "compile:windows": "bun build --compile --minify --bytecode --target=bun-windows-x64 --define VERSION='\"0.1.0\"' ./server.ts --outfile ./dist/cortex-mem-insights.exe", - "compile:all": "bun run compile:mac && bun run compile:mac-x64 && bun run compile:linux && bun run compile:windows" - }, - "devDependencies": { - "@sveltejs/vite-plugin-svelte": "^5.0.3", - "svelte": "^5.19.0", - "svelte-check": "^4.1.4", - "typescript": "~5.7.3", - "vite": "^6.1.0" - }, - "dependencies": { - "svelte-routing": "^2.13.0" - } + "name": "cortex-mem-insights", + "private": true, + "version": "2.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "serve": "bun server.ts", + "compile": "bun build --compile --minify --bytecode --sourcemap ./server.ts --outfile ./dist/cortex-mem-insights", + "compile:prod": "bun build --compile --minify --bytecode --sourcemap --define VERSION='\"2.0.0\"' --define BUILD_TIME='\"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'\"' ./server.ts --outfile ./dist/cortex-mem-insights", + "compile:mac": "bun build --compile --minify --bytecode --target=bun-darwin-arm64 --define VERSION='\"2.0.0\"' ./server.ts --outfile ./dist/cortex-mem-insights-mac-arm64", + "compile:mac-x64": "bun build --compile --minify --bytecode --target=bun-darwin-x64 --define VERSION='\"2.0.0\"' ./server.ts --outfile ./dist/cortex-mem-insights-mac-x64", + "compile:linux": "bun build --compile --minify --bytecode --target=bun-linux-x64 --define VERSION='\"2.0.0\"' ./server.ts --outfile ./dist/cortex-mem-insights-linux", + "compile:windows": "bun build --compile --minify --bytecode --target=bun-windows-x64 --define VERSION='\"2.0.0\"' ./server.ts --outfile ./dist/cortex-mem-insights.exe", + "compile:all": "bun run compile:mac && bun run compile:mac-x64 && bun run compile:linux && bun run compile:windows" + }, + "devDependencies": { + "@sveltejs/vite-plugin-svelte": "^5.0.3", + "svelte": "^5.19.0", + "svelte-check": "^4.1.4", + "typescript": "~5.7.3", + "vite": "^6.1.0" + }, + "dependencies": { + "svelte-routing": "^2.13.0" + } } diff --git a/cortex-mem-service/Cargo.toml b/cortex-mem-service/Cargo.toml index 77ddad4..132d598 100644 --- a/cortex-mem-service/Cargo.toml +++ b/cortex-mem-service/Cargo.toml @@ -4,7 +4,7 @@ version = "2.0.0" edition = "2024" authors.workspace = true license.workspace = true -description = "HTTP REST API service for Cortex-Mem V2" +description = "HTTP REST API service for Cortex Memory" [[bin]] name = "cortex-mem-service" diff --git a/cortex-mem-service/src/main.rs b/cortex-mem-service/src/main.rs index 5556f2a..d2a720c 100644 --- a/cortex-mem-service/src/main.rs +++ b/cortex-mem-service/src/main.rs @@ -17,7 +17,7 @@ use state::AppState; #[derive(Parser, Debug)] #[command(name = "cortex-mem-service")] -#[command(about = "Cortex-Mem V2 HTTP REST API Service", long_about = None)] +#[command(about = "Cortex Memory HTTP REST API Service", long_about = None)] #[command(version)] struct Cli { /// Data directory for cortex filesystem diff --git a/examples/cortex-mem-tars/config.example.toml b/examples/cortex-mem-tars/config.example.toml index ae16580..ff640d6 100644 --- a/examples/cortex-mem-tars/config.example.toml +++ b/examples/cortex-mem-tars/config.example.toml @@ -1,5 +1,5 @@ -# Cortex-Mem V2 配置文件 (简化版) -# +# Cortex Memory 配置文件 (简化版) +# # 架构说明： # - 当前使用：文件系统存储 + 关键词检索 # - 保留向量配置：为未来向量搜索功能预留 @@ -52,7 +52,7 @@ cors_origins = ["*"] # - Linux: ~/.local/share/cortex-mem-tars/cortex # - Windows: %APPDATA%\cortex-mem\tars\cortex # 4. 当前目录 ./.cortex（最低优先级） -# +# # 留空或注释此行将使用默认值（应用数据目录） # data_dir = "/path/to/custom/cortex/data"