From 4a7e1af31457c7f72c534b618d76c51b93919fd9 Mon Sep 17 00:00:00 2001
From: thiswillbeyourgithub
 <26625900+thiswillbeyourgithub@users.noreply.github.com>
Date: Tue, 23 Apr 2024 13:48:53 +0200
Subject: [PATCH] fix: vision models

Signed-off-by: thiswillbeyourgithub <26625900+thiswillbeyourgithub@users.noreply.github.com>
---
 README.md                       |  3 +++
 lua/chatgpt/flows/chat/base.lua | 12 ++++++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 5371a6bc..a6bcc9e3 100644
--- a/README.md
+++ b/README.md
@@ -32,6 +32,9 @@ test addition, code optimization, summarization, bug fixing, code explanation,
 Roxygen editing, and code readability analysis. Additionally, you can define
 your own custom actions using a JSON file.
 
+- **Partial Vision Support**: Any url linking to an image will be sent as an
+image if the model supports vision.
+
 For a comprehensive understanding of the extension's functionality, you can watch
 a plugin showcase [video](https://www.youtube.com/watch?v=7k0KZsheLP4)
 
diff --git a/lua/chatgpt/flows/chat/base.lua b/lua/chatgpt/flows/chat/base.lua
index f37e1386..0d167ff7 100644
--- a/lua/chatgpt/flows/chat/base.lua
+++ b/lua/chatgpt/flows/chat/base.lua
@@ -497,7 +497,7 @@ local function createContent(line)
   local extensions = { "%.jpeg", "%.jpg", "%.png", "%.gif", "%.bmp", "%.tif", "%.tiff", "%.webp" }
   for _, ext in ipairs(extensions) do
     if string.find(line:lower(), ext .. "$") then
-      return { type = "image_url", image_url = line }
+      return { type = "image_url", image_url = { url = line } }
     end
   end
   return { type = "text", text = line }
@@ -505,10 +505,18 @@ end
 
 function Chat:toMessages()
   local messages = {}
+  local use_vision = false
   if self.system_message ~= nil then
     table.insert(messages, { role = "system", content = self.system_message })
   end
 
+  if string.find(self.params.model, "vision", 1, true) or
+        string.find(self.params.model, "gpt-4-turbo", 1, true) or
+        string.find(Settings.params.model, "vision", 1, true) or
+        string.find(Settings.params.model, "gpt-4-turbo", 1, true) then
+      use_vision = true
+  end
+
   for _, msg in pairs(self.messages) do
     local role = "user"
     if msg.type == SYSTEM then
@@ -517,7 +525,7 @@ function Chat:toMessages()
       role = "assistant"
     end
     local content = {}
-    if self.params.model == "gpt-4-vision-preview" then
+    if use_vision then
       for _, line in ipairs(msg.lines) do
         table.insert(content, createContent(line))
       end