Chatbot with langchain vector store

.env.sample

@@ -5,7 +5,7 @@
 
 # Use DEFAULT for the meta portal
 NUXT_PUBLIC_APP_API_ORG_CODE=CRI
-# Custom favicon 
+# Custom favicon
 # (can be an absolute path, else will be relative to NUXT_PUBLIC_APP_PUBLIC_BINARIES_PREFIX)
 NUXT_PUBLIC_APP_FAVICON=favicon.ico
 # url prefix for a binary file storage (images...)
@@ -77,7 +77,7 @@ PLAYWRIGHT_TEST_PASSWORD=
 ### examples suggestions separated by § character
 # NUXT_PUBLIC_APP_CHATBOT_EXEMPLES=Tell me about this platform§Find research projects on renewable energy§Find researchers and publication on climate change
 
-# ==== OpenAI PRIVATE settings ==== 
+# ==== OpenAI PRIVATE settings ====
 # OpenAI API Key for the chatbot
 # optional
 # NUXT_APP_OPENAI_API_KEY=sk-proj-XXXXXXXXXXXXXXXXXXXXXXXXXXXXX
@@ -96,4 +96,11 @@ PLAYWRIGHT_TEST_PASSWORD=
 
 # ==== GOTENBERG pdf (optional) Settings ====
 # NUXT_APP_GOTENBERG_SERVER_URL=http://127.0.0.1:3333
-# NUXT_PUBLIC_APP_GOTENBERG_ENABLED='true'
+# NUXT_PUBLIC_APP_GOTENBERG_ENABLED='true'
+
+# ==== VECTOR DATABASE (optional) Settings ====
+# NUXT_APP_VECTOR_DB_URL=postgresql://myuser:mypassword@localhost:5432/mydb
+# NUXT_APP_VECTOR_EMBEDDING_MODEL=text-embedding-3-small
+# NUXT_APP_VECTOR_EMBEDDING_API_KEY=XXXXX
+# NUXT_APP_VECTOR_EMBEDDING_DIMENSIONS=1536
+# NUXT_APP_VECTOR_TOOL_PROMPT="use this tool to answer about neuroscience project"

k8s/projects-frontend/values.dev.yaml

@@ -1,11 +1,16 @@
 config:
   nonSensitive:
     NUXT_PUBLIC_APP_KEYCLOAK_URL: https://keycloak.k8s.lp-i.dev
-    NUXT_PUBLIC_APP_VERSION: "{{ .Values.application.revision }}"
+    NUXT_PUBLIC_APP_VERSION: '{{ .Values.application.revision }}'
     NUXT_PUBLIC_APP_ENVIRONMENT: dev
     NUXT_APP_GOTENBERG_SERVER_URL: 'http://projects-gotenberg'
     NUXT_PUBLIC_APP_GOTENBERG_ENABLED: 'true'
     NUXT_APP_LANGCHAIN_PROMPT: ''
+    NUXT_APP_VECTOR_DB_URL: ''
+    NUXT_APP_VECTOR_EMBEDDING_API_KEY: ''
+    NUXT_APP_VECTOR_EMBEDDING_MODEL: ''
+    NUXT_APP_VECTOR_EMBEDDING_DIMENSIONS: ''
+    NUXT_APP_VECTOR_TOOL_PROMPT: ''
 
 domain: k8s.lp-i.dev
 

k8s/projects-frontend/values.yaml

@@ -45,6 +45,11 @@ config:
     NUXT_APP_GOTENBERG_SERVER_URL: ''
     NUXT_PUBLIC_APP_GOTENBERG_ENABLED: ''
     NUXT_APP_LANGCHAIN_PROMPT: ''
+    NUXT_APP_VECTOR_DB_URL: ''
+    NUXT_APP_VECTOR_EMBEDDING_API_KEY: ''
+    NUXT_APP_VECTOR_EMBEDDING_MODEL: ''
+    NUXT_APP_VECTOR_EMBEDDING_DIMENSIONS: ''
+    NUXT_APP_VECTOR_TOOL_PROMPT: ''
 e2eEnv:
   nonSensitive:
     USER_ADMIN_EMAIL: testautomatatiquedministrateur1@outlook.fr

nuxt.config.ts

@@ -131,14 +131,19 @@ export default defineNuxtConfig({
     appGeocodingApiKey: '',
     appOpenaiApiKey: '',
     appOpenaiApiPromptId: '',
-    appOpenaiApiPromptVersion: '',
+    appOpeyanaiApiPromptVersion: '',
     appOpenaiApiVectorStoreId: '',
     appMcpServerUrl: '',
     appSorbobotApiToken: '',
     appMcpServerTrace: 0,
     appSorbobotApiTrace: 0,
     appGotenbergServerUrl: '',
     appLangchainPrompt: '',
+    appVectorDbUrl: '',
+    appVectorEmbeddingApiKey: '',
+    appVectorEmbeddingModel: '',
+    appVectorEmbeddingDimensions: '',
+    appVectorToolPrompt: '',
     public: {
       appVersion: '',
       appApiOrgCode: '',

package.json

@@ -42,9 +42,11 @@
     "@ckpack/vue-color": "^1.5.0",
     "@hocuspocus/provider": "^2.15.0",
     "@intlify/vue-i18n-loader": "^4.2.0",
-    "@langchain/core": "^1.1.17",
+    "@langchain/community": "^1.1.23",
+    "@langchain/core": "^1.1.32",
     "@langchain/langgraph": "^1.1.2",
     "@langchain/openai": "^1.2.3",
+    "@langchain/textsplitters": "^1.0.1",
     "@mdi/font": "^6",
     "@modelcontextprotocol/sdk": "^1.20.2",
     "@nuxt/test-utils": "^3.19.2",
@@ -84,6 +86,7 @@
     "deep-chat": "^2.3.0",
     "es-toolkit": "^1.40.0",
     "highlight.js": "^11",
+    "install": "^0.13.0",
     "langchain": "^1.2.14",
     "leaflet": "^1",
     "leaflet.markercluster": "^1.5.3",
@@ -92,6 +95,8 @@
     "nuxt": "^3.19.3",
     "nuxt-svgo": "4.2.6",
     "openai": "^5.23.0",
+    "pdf-parse": "^1",
+    "pg": "^8.20.0",
     "pinia": "^3.0.3",
     "prosemirror-markdown": "^1.13.1",
     "prosemirror-model": "^1.24.1",

src/pages/DebugPage/Tabs/DebugOnboarding.vue

@@ -19,6 +19,22 @@
       :disabled="!usersStore.userFromApi || resetingTerms"
       @click="resetTermsSigned"
     />
+    <hr />
+
+    <form
+      target="_blank"
+      action="/api/vector-store/ingest"
+      method="POST"
+      enctype="multipart/form-data"
+    >
+      <label for="title">Title</label>
+      <input id="title" type="text" name="title" required />
+
+      <label for="file">File</label>
+      <input id="file" type="file" name="file" required />
+
+      <button type="submit">Upload</button>
+    </form>
   </div>
 </template>
 <script>

src/server/routes/api/chat-lg-stream.ts

@@ -1,19 +1,24 @@
 // import OpenAI from 'openai'
 import { ChatOpenAI } from '@langchain/openai'
-import { createAgent } from 'langchain'
+import { createAgent, createMiddleware } from 'langchain'
 import { MemorySaver } from '@langchain/langgraph'
-import { SystemMessage, HumanMessage } from '@langchain/core/messages'
+import { SystemMessage, HumanMessage, AIMessage } from '@langchain/core/messages'
 import { v4 as uuidv4 } from 'uuid'
+import getVectorStore from '@/server/utils/vector-db.js'
+import { tool } from '@langchain/core/tools'
+import { createRetrieverTool } from '@langchain/classic/tools/retriever'
+
 const runtimeConfig = useRuntimeConfig()
 const {
   // appOpenaiApiPromptId,
   // appOpenaiApiPromptVersion,
   appOpenaiApiKey,
   appOpenaiApiVectorStoreId,
   appMcpServerUrl,
   appMcpServerTrace,
   appSorbobotApiTrace,
   appLangchainPrompt,
+  appVectorToolPrompt,
 } = runtimeConfig
 const { appChatbotEnabled } = runtimeConfig.public
 
@@ -96,17 +101,71 @@
       })
     }
 
-    if (appOpenaiApiVectorStoreId) {
-      tools.push({
-        type: 'file_search',
-        vector_store_ids: [appOpenaiApiVectorStoreId],
+    // if (appOpenaiApiVectorStoreId) {
+    //   tools.push({
+    //     // name: 'openai_vectorstore',
+    //     type: 'file_search',
+    //     vector_store_ids: [appOpenaiApiVectorStoreId],
+    //   })
+    // }
+
+    const vectorStore = await getVectorStore()
+    if (vectorStore) {
+      console.log(`Configure vector tool with prompt "${appVectorToolPrompt}"`)
+      const retriever = vectorStore.asRetriever({ k: 5 }) // Top 4 similar docs
+      // Create tool from retriever
+      // const retrieverTool = tool(
+      //   async (query) => {
+      //     console.log(query)
+      //     const docs = await retriever.invoke(query)
+      //     return docs.map((doc: Document) => doc.pageContent).join('\n\n')
+      //   },
+      //   {
+      //     name: 'pgvector_search',
+      //     type: 'file_search',
+      //     // description: 'Search for information in the document database. Use this tool when you need to answer questions about the uploaded content.',
+      //     description: appVectorToolPrompt,
+      //     schema: {
+      //       type: 'object',
+      //       properties: {
+      //         query: {
+      //           type: 'string',
+      //           description: 'Search query',
+      //         },
+      //       },
+      //       required: ['query'],
+      //     },
+      //     require_approval: 'never',
+      //   }
+      // )
+      const retrieverTool = createRetrieverTool(retriever, {
+        name: 'pgvector_tool',
+        description: appVectorToolPrompt,
       })
+      tools.push(retrieverTool)
     }
+
+    const toolMonitoringMiddleware = createMiddleware({
+      name: 'ToolMonitoringMiddleware',
+      wrapToolCall: (request, handler) => {
+        console.log(`Executing tool: ${request.toolCall.name}`)
+        console.log(`Arguments: ${JSON.stringify(request.toolCall.args)}`)
+        try {
+          const result = handler(request)
+          console.log('Tool completed successfully')
+          return result
+        } catch (e) {
+          console.log(`Tool failed: ${e}`)
+          throw e
+        }
+      },
+    })
+
     const model = appOpenaiApiKey
       ? new ChatOpenAI({
           apiKey: appOpenaiApiKey,
           model: 'gpt-4o-mini',
-          temperature: 0,
+          temperature: 0.2,
         })
       : null
 
@@ -118,14 +177,15 @@
         content: [
           {
             type: 'text',
-            content: appLangchainPrompt,
+            text: appLangchainPrompt,
           },
         ],
       }),
+      middleware: [toolMonitoringMiddleware] as any,
     })
 
     traceMcp(
-      `Starting chat stream for conversation ${conversationId} with ${messages.length} messages`
+      `Starting chat stream for conversation ${conversationId} with ${tools.map((t) => `"${t.name}"`).join(', ')}  tools and ${messages.length} messages`
     )
 
     tokenMap.set(conversationId, {
@@ -169,7 +229,14 @@
   */
     // TODO: fix typescript mess with agent.stream return type
     for await (const [token, metadata] of (await agent.stream(
-      { messages: messages.map((msg) => new HumanMessage(msg.text)) } as any,
+      // { messages: messages.map((msg) => new HumanMessage(msg.text)) } as any,
+      {
+        messages: messages.map((msg) =>
+          msg.role === 'ai' || msg.role === 'assistant'
+            ? new AIMessage(msg.text)
+            : new HumanMessage(msg.text)
+        ),
+      } as any,
       { ...config, streamMode: 'messages' }
       // ,{ options: { stream: true }, previous_response_id: conversationId,}
     )) as AsyncIterableIterator<

src/server/routes/api/vector-store/ingest.ts

@@ -0,0 +1,59 @@
+import getVectorStore from '@/server/utils/vector-db.js'
+import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf'
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters'
+
+export default defineLazyEventHandler(() => {
+  return defineEventHandler(async (event) => {
+    const { appApiOrgCode } = useRuntimeConfig().public
+    const vectorStore = await getVectorStore()
+    // return 404 if not configured
+    if (!vectorStore) {
+      setResponseStatus(event, 404)
+      return
+    }
+
+    const formData = await readFormData(event)
+
+    const file = formData.get('file') as File
+    const title = formData.get('title') as string
+
+    if (!file || !title) {
+      throw createError({
+        statusCode: 400,
+        message: 'Missing required fields: file and title',
+      })
+    }
+
+    // const arrayBuffer = await file.arrayBuffer()
+    // const buffer = Buffer.from(arrayBuffer)
+    const loader = new PDFLoader(file)
+
+    const fileDocs = await loader.load()
+    const extraMetadata = {
+      title: title,
+      timestamp: new Date().toISOString(),
+      orgCode: appApiOrgCode,
+    }
+    const fileDocsWithMeta = fileDocs.map((d) => ({
+      ...d,
+      metadata: { ...d.metadata, ...extraMetadata },
+    }))
+
+    // chunking express
+    const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 500, chunkOverlap: 100 })
+    const chunks = await splitter.splitDocuments(fileDocsWithMeta)
+
+    await vectorStore.delete({
+      filter: {
+        metadata: {
+          title: extraMetadata.title,
+          orgCode: extraMetadata.orgCode,
+        },
+      },
+    })
+
+    await vectorStore.addDocuments(chunks)
+
+    return { status: 'ok', chunks }
+  })
+})

src/server/utils/vector-db.js

@@ -0,0 +1,68 @@
+import { PGVectorStore } from '@langchain/community/vectorstores/pgvector'
+import { OpenAIEmbeddings } from '@langchain/openai' // Or any other embedding model
+// import { Client } from 'pg'
+import { parse } from 'pg-connection-string'
+
+export default async () => {
+  const runtimeConfig = useRuntimeConfig()
+  // 1. Configure the database connection
+  const connectionString = runtimeConfig.appVectorDbUrl
+  const modelName = runtimeConfig.appVectorEmbeddingModel
+  const vectorDimensions = runtimeConfig.appVectorEmbeddingDimensions
+    ? parseInt(runtimeConfig.appVectorEmbeddingDimensions)
+    : null
+  const apiKey = runtimeConfig.appVectorEmbeddingApiKey
+
+  let vectorStore = null
+  try {
+    if (!connectionString || !modelName || !vectorDimensions || !apiKey) {
+      throw new Error('Missing required configuration for vector store.')
+    }
+
+    // 2. Create a PgPool or Client instance (LangChain accepts both, but Pool is better for production)
+    // For simplicity in initialization, we'll pass the connection string directly to the store constructor
+    // which handles the connection internally for setup.
+
+    const embeddings = new OpenAIEmbeddings({
+      modelName,
+      apiKey,
+    })
+
+    const config = {
+      postgresConnectionOptions: parse(connectionString),
+      dimensions: vectorDimensions,
+      tableName: 'testlangchain',
+      columns: {
+        idColumnName: 'id',
+        vectorColumnName: 'vector',
+        contentColumnName: 'content',
+        metadataColumnName: 'metadata',
+      },
+      // supported distance strategies: cosine (default), innerProduct, or euclidean
+      distanceStrategy: 'cosine', // as DistanceStrategy,
+    }
+
+    // 3. Initialize the VectorStore - skip if not needed
+    vectorStore = await PGVectorStore.initialize(
+      embeddings,
+      config
+      // Optional: Specify a custom table name if you don't want the default 'langchain_pg_embedding'
+      // tableName: "my_custom_table",
+      // Optional: Use JSONB for metadata (default is usually JSONB in newer versions)
+      // useJsonb: true,
+    )
+
+    console.log('Database connection established and tables initialized.')
+
+    // 4. (Optional) Add some documents to verify it works
+    // const docs = [
+    //   new Document({ pageContent: "Hello world", metadata: { source: "test" } }),
+    // ];
+    // await vectorStore.addDocuments(docs);
+  } catch (error) {
+    console.error('Error initializing vector store:', error)
+    // Handle the error as needed, e.g., fallback to an in-memory store or disable vector features
+  }
+
+  return vectorStore
+}