feat: redesign embeddings interface with LlamaIndex

pyproject.toml

@@ -16,7 +16,7 @@ llama-cpp-python = "^0.2.26"
 litellm = "^1.23.12"
 txtai = {version = "^7.0.0", extras = ["pipeline-llm"]}
 sse-starlette = "^2.0.0"
-llama-index = "^0.10.4"
+llama-index = "^0.10.18"
 numpy = "^1.26.4"
 html2text = "^2020.1.16"
 peewee = "^3.17.0"
@@ -27,6 +27,10 @@ python-dotenv = "^1.0.1"
 pillow = "^10.2.0"
 swig = "^4.2.1"
 
+llama-index-vector-stores-txtai = "^0.1.2"
+llama-index-llms-openai-like = "^0.1.3"
+llama-index-embeddings-huggingface = "^0.1.4"
+
 # Experimental GPU Features
 auto-gptq = { version = "^0.6.0", optional = true }
 optimum = { version = "^1.16.2", optional = true }

selfie/api/completions.py

@@ -4,6 +4,12 @@
     CreateCompletionResponse as LlamaCppCompletionResponse,
     CreateChatCompletionResponse as LlamaCppChatCompletionResponse,
 )
+from llama_index.core import VectorStoreIndex, ServiceContext
+from llama_index.core.vector_stores import VectorStoreQuery
+from llama_index.llms.openai_like import OpenAILike
+
+from selfie.embeddings.TxtaiVectorStore import TxtaiVectorStore
+from selfie.embeddings import DataIndex
 from selfie.types.completion_requests import ChatCompletionRequest, CompletionRequest
 
 from selfie.text_generation import completion

selfie/api/documents.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from typing import List, Optional
 
 from fastapi import APIRouter, Query
@@ -27,8 +28,8 @@ class FetchedDocument(BaseModel):
     id: int = Field(..., description="The unique identifier of the document")
     name: str = Field(..., description="The name of the document")
     size: int = Field(..., description="The size of the document")
-    created_at: str = Field(..., description="The timestamp of the document creation")
-    updated_at: str = Field(..., description="The timestamp of the document update")
+    created_at: datetime = Field(..., description="The timestamp of the document creation")
+    updated_at: datetime = Field(..., description="The timestamp of the document update")
     content_type: str = Field(..., description="The content type of the document")
     connector_name: str = Field(..., description="The name of the connector")
 

selfie/api/index_documents.py

@@ -53,7 +53,7 @@ async def update_index_document(document_id: int, document: EmbeddingDocumentMod
 async def delete_index_document(document_id: int):
     # Sometimes self.embeddings.save() errors on "database is locked", bricks it
     # raise HTTPException(status_code=501, detail="Not implemented")
-    DataIndex("n/a").delete_document(document_id)
+    await DataIndex("n/a").delete_document(document_id)
     return {"message": "Document deleted successfully"}
 
 

selfie/connectors/text_files/uischema.json

@@ -1,8 +1,5 @@
 {
   "files": {
-    "ui:widget": "nativeFile",
-    "ui:options": {
-      "accept": ".json"
-    }
+    "ui:widget": "nativeFile"
   }
 }