From c0ad2b8562bf2c3b367a52dfdea0e6d255f727af Mon Sep 17 00:00:00 2001
From: Diljit <diljit@appsmith.com>
Date: Mon, 17 Jun 2024 14:07:03 +0530
Subject: [PATCH] chore: Add gpt-4o mode in vision command (#33637)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Description
Add gpt-4o model in Open AI datasource's vision and chat commands

Fixes #`Issue Number`
_or_
Fixes `Issue URL`
> [!WARNING]
> _If no issue exists, please create an issue first, and check with the
maintainers if the issue is valid._

## Automation

/ok-to-test tags="@tag.All"

### :mag: Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!TIP]
> 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/9541802412>
> Commit: 78eb94443c7e8ed5396f6e3369aac1e5373954bc
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=9541802412&attempt=1"
target="_blank">Cypress dashboard</a>.
> Tags: ``

<!-- end of auto-generated comment: Cypress test results  -->










## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [ ] No


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **New Features**
- Enhanced `VisionCommand` to support new model naming conventions,
including `gpt-4-vision-preview` and `gpt-4o`.

- **Bug Fixes**
- Improved message content handling in `transformUserMessages` and
`transformSystemMessages` methods to ensure proper data type casting.

- **Tests**
- Updated test cases to align with the new model name `gpt-4o`, ensuring
accurate test coverage.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Nirmal Sarswat <nirmal@appsmith.com>
---
 .../com/external/plugins/commands/VisionCommand.java     | 9 +++++----
 .../java/com/external/plugins/models/VisionMessage.java  | 4 +---
 .../java/com/external/plugins/VisionCommandTest.java     | 2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/VisionCommand.java b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/VisionCommand.java
index dd44c076d8..f7416ba5c6 100644
--- a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/VisionCommand.java
+++ b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/VisionCommand.java
@@ -60,7 +60,8 @@ public class VisionCommand implements OpenAICommand {
 
     private final Gson gson;
 
-    private final String regex = "(ft:)?(gpt).*(vision).*";
+    private final String regex =
+            "^(gpt-4-(vision-preview|\\d{4}-vision-preview)|gpt-4o(.*)?|ft:(gpt-4-(vision-preview|\\d{4}-vision-preview)|gpt-4o).*)$";
     private final Pattern pattern = Pattern.compile(regex);
 
     @Override
@@ -135,12 +136,12 @@ public class VisionCommand implements OpenAICommand {
                     UserTextContent userContent = new UserTextContent();
                     userContent.setType(TEXT_TYPE);
                     userContent.setText(userQuery.getContent());
-                    visionMessage.getContent().add(userContent);
+                    ((List<Object>) visionMessage.getContent()).add(userContent);
                 } else if (QueryType.IMAGE.equals(userQuery.getType())) {
                     UserImageContent userContent = new UserImageContent();
                     userContent.setType(IMAGE_TYPE);
                     userContent.setImageUrl(new UserImageContent.ImageUrl(userQuery.getContent()));
-                    visionMessage.getContent().add(userContent);
+                    ((List<Object>) visionMessage.getContent()).add(userContent);
                 }
             }
             return List.of(visionMessage);
@@ -168,7 +169,7 @@ public class VisionCommand implements OpenAICommand {
                 VisionMessage visionMessage = new VisionMessage();
                 if (StringUtils.hasText(systemMessageMap.get(CONTENT))) {
                     visionMessage.setRole(Role.system);
-                    visionMessage.setContent(List.of(systemMessageMap.get(CONTENT)));
+                    visionMessage.setContent(systemMessageMap.get(CONTENT));
                     visionMessages.add(visionMessage);
                 }
             }
diff --git a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/models/VisionMessage.java b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/models/VisionMessage.java
index 1c33952661..eeaf6502d7 100644
--- a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/models/VisionMessage.java
+++ b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/models/VisionMessage.java
@@ -4,12 +4,10 @@ import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 
-import java.util.List;
-
 @Getter
 @Setter
 @NoArgsConstructor
 public class VisionMessage {
     Role role;
-    List<Object> content;
+    Object content;
 }
diff --git a/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/VisionCommandTest.java b/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/VisionCommandTest.java
index f22d27b585..983b667d3c 100644
--- a/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/VisionCommandTest.java
+++ b/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/VisionCommandTest.java
@@ -144,7 +144,7 @@ public class VisionCommandTest {
                 "curie",
                 "davinci",
                 "gpt-4-vision-preview",
-                "gpt-4.5-vision-preview");
+                "gpt-4o");
         int counter = 0;
         for (String model : models) {
             JSONObject jsonObject = new JSONObject(String.format("{\"%s\": \"%s\" }", ID, model));