From b7c8c2b39678711293f3f6f66921faa8be94f3cc Mon Sep 17 00:00:00 2001 From: Nirmal Sarswat <25587962+vivonk@users.noreply.github.com> Date: Fri, 22 Dec 2023 13:55:56 +0530 Subject: [PATCH] feat: Enable JS toggle on messages in OpenAI plugin (#29519) ## Description Enable JS toggle on top of messages in Chat and Vision command/API integration. #### PR fixes following issue(s) Fixes https://github.com/appsmithorg/appsmith/issues/29220 #### Type of change - New feature (non-breaking change which adds functionality) ## Testing #### How Has This Been Tested? - [x] Manual - [x] JUnit - [ ] Jest - [ ] Cypress ## Checklist: #### Dev activity - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] PR is being merged under a feature flag #### QA activity: - [ ] [Speedbreak features](https://github.com/appsmithorg/TestSmith/wiki/Guidelines-for-test-plans#speedbreakers-) have been covered - [ ] Test plan covers all impacted features and [areas of interest](https://github.com/appsmithorg/TestSmith/wiki/Guidelines-for-test-plans#areas-of-interest-) - [ ] Test plan has been peer reviewed by project stakeholders and other QA members - [ ] Manually tested functionality on DP - [ ] We had an implementation alignment call with stakeholders post QA Round 2 - [ ] Cypress test cases have been added and approved by SDET/manual QA - [ ] Added `Test Plan Approved` label after Cypress tests were reviewed - [ ] Added `Test Plan Approved` label after JUnit tests were reviewed ## Summary by CodeRabbit - **New Features** - Enhanced chat and vision command functionalities to support new message types and data structures. - Introduced new constants to standardize data handling across plugins. - **Refactor** - Streamlined message extraction logic using a shared utility method for improved consistency. - **Bug Fixes** - Adjusted the handling of message data in test cases to align with updated logic. - **Documentation** - Updated internal documentation to reflect changes in constants and message handling methods. - **Chores** - Performed database migration to restructure message data, enabling support for a new JavaScript toggle feature in the UI. --- .../plugins/commands/ChatCommand.java | 4 +- .../plugins/commands/VisionCommand.java | 7 +- .../plugins/constants/OpenAIConstants.java | 4 + .../external/plugins/utils/MessageUtils.java | 36 +++++++ .../src/main/resources/editor/chat.json | 18 +++- .../src/main/resources/editor/vision.json | 6 +- .../com/external/plugins/ChatCommandTest.java | 2 +- .../external/plugins/VisionCommandTest.java | 5 +- .../Migration039OpenAIMessagesJsToggle.java | 95 +++++++++++++++++++ 9 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/utils/MessageUtils.java create mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration039OpenAIMessagesJsToggle.java diff --git a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/ChatCommand.java b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/ChatCommand.java index 8007be09d7..e22e09bac1 100644 --- a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/ChatCommand.java +++ b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/ChatCommand.java @@ -6,6 +6,7 @@ import com.appsmith.external.models.ActionConfiguration; import com.external.plugins.models.ChatMessage; import com.external.plugins.models.ChatRequestDTO; import com.external.plugins.models.OpenAIRequestDTO; +import com.external.plugins.utils.MessageUtils; import com.external.plugins.utils.RequestUtils; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; @@ -90,7 +91,8 @@ public class ChatCommand implements OpenAICommand { chatRequestDTO.setModel(model); // this will change to objects - List chatMessages = transformToMessages(formData.get(MESSAGES)); + List chatMessages = + transformToMessages(MessageUtils.extractMessages((Map) formData.get(MESSAGES))); verifyRoleForChatMessages(chatMessages); Float temperature = getTemperatureFromFormData(formData); diff --git a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/VisionCommand.java b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/VisionCommand.java index 31e0e31537..dd44c076d8 100644 --- a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/VisionCommand.java +++ b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/commands/VisionCommand.java @@ -11,6 +11,7 @@ import com.external.plugins.models.UserQuery; import com.external.plugins.models.UserTextContent; import com.external.plugins.models.VisionMessage; import com.external.plugins.models.VisionRequestDTO; +import com.external.plugins.utils.MessageUtils; import com.external.plugins.utils.RequestUtils; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; @@ -102,8 +103,10 @@ public class VisionCommand implements OpenAICommand { visionRequestDTO.setModel(model); - List visionMessages = transformSystemMessages(formData.get(SYSTEM_MESSAGES)); - visionMessages.addAll(transformUserMessages(formData.get(USER_MESSAGES))); + List visionMessages = transformSystemMessages( + MessageUtils.extractMessages((Map) formData.get(SYSTEM_MESSAGES))); + visionMessages.addAll( + transformUserMessages(MessageUtils.extractMessages((Map) formData.get(USER_MESSAGES)))); Float temperature = getTemperatureFromFormData(formData); visionRequestDTO.setMessages(visionMessages); diff --git a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/constants/OpenAIConstants.java b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/constants/OpenAIConstants.java index 7a3f7069f4..d44c21673a 100644 --- a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/constants/OpenAIConstants.java +++ b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/constants/OpenAIConstants.java @@ -30,6 +30,8 @@ public class OpenAIConstants { public static final String COMMAND = "command"; public static final String MODEL = "model"; public static final String MESSAGES = "messages"; + public static final String VIEW_TYPE = "viewType"; + public static final String COMPONENT_DATA = "componentData"; public static final String SYSTEM_MESSAGES = "systemMessages"; public static final String USER_MESSAGES = "userMessages"; public static final String MAX_TOKENS = "maxTokens"; @@ -43,6 +45,8 @@ public class OpenAIConstants { // Other constants public static final String BODY = "body"; public static final Integer DEFAULT_MAX_TOKEN = 16; + public static final String JSON = "json"; + public static final String COMPONENT = "component"; public static final ExchangeStrategies EXCHANGE_STRATEGIES = ExchangeStrategies.builder() .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(/* 10MB */ 10 * 1024 * 1024)) diff --git a/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/utils/MessageUtils.java b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/utils/MessageUtils.java new file mode 100644 index 0000000000..78cd4efe97 --- /dev/null +++ b/app/server/appsmith-plugins/openAiPlugin/src/main/java/com/external/plugins/utils/MessageUtils.java @@ -0,0 +1,36 @@ +package com.external.plugins.utils; + +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; +import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.springframework.util.CollectionUtils; + +import java.lang.reflect.Type; +import java.util.List; +import java.util.Map; + +import static com.external.plugins.constants.OpenAIConstants.DATA; +import static com.external.plugins.constants.OpenAIConstants.JSON; +import static com.external.plugins.constants.OpenAIConstants.VIEW_TYPE; +import static com.external.plugins.constants.OpenAIErrorMessages.EXECUTION_FAILURE; +import static com.external.plugins.constants.OpenAIErrorMessages.INCORRECT_MESSAGE_FORMAT; +import static com.external.plugins.constants.OpenAIErrorMessages.STRING_APPENDER; + +public class MessageUtils { + private static final Gson gson = new Gson(); + + public static Object extractMessages(Map messages) { + if (CollectionUtils.isEmpty(messages) || !messages.containsKey(DATA)) { + throw new AppsmithPluginException( + AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, + String.format(STRING_APPENDER, EXECUTION_FAILURE, INCORRECT_MESSAGE_FORMAT)); + } + Type listType = new TypeToken>>() {}.getType(); + if (messages.containsKey(VIEW_TYPE) && JSON.equals(messages.get(VIEW_TYPE))) { + // data is present in data key as String + return gson.fromJson((String) messages.get(DATA), listType); + } + return messages.get(DATA); + } +} diff --git a/app/server/appsmith-plugins/openAiPlugin/src/main/resources/editor/chat.json b/app/server/appsmith-plugins/openAiPlugin/src/main/resources/editor/chat.json index 280434e921..c29af65c27 100644 --- a/app/server/appsmith-plugins/openAiPlugin/src/main/resources/editor/chat.json +++ b/app/server/appsmith-plugins/openAiPlugin/src/main/resources/editor/chat.json @@ -32,14 +32,30 @@ } } }, + { + "label": "Max Tokens", + "tooltipText": "The maximum number of tokens to generate in the chat completion.", + "subtitle": "The maximum number of tokens to generate in the chat completion.", + "Description": "Put a positive integer value", + "configProperty": "actionConfiguration.formData.maxTokens", + "controlType": "INPUT_TEXT", + "initialValue": "16", + "isRequired": true, + "dataType": "NUMBER", + "customStyles": { + "width": "270px", + "minWidth": "270px" + } + }, { "label": "Messages", "tooltipText": "Ask a question", "subtitle": "A list of messages comprising the conversation so far.", "propertyName": "messages", "isRequired": true, - "configProperty": "actionConfiguration.formData.messages", + "configProperty": "actionConfiguration.formData.messages.data", "controlType": "ARRAY_FIELD", + "alternateViewTypes": ["json"], "addMoreButtonLabel": "Add message", "schema": [ { diff --git a/app/server/appsmith-plugins/openAiPlugin/src/main/resources/editor/vision.json b/app/server/appsmith-plugins/openAiPlugin/src/main/resources/editor/vision.json index 7888cf460f..79ed3f4742 100644 --- a/app/server/appsmith-plugins/openAiPlugin/src/main/resources/editor/vision.json +++ b/app/server/appsmith-plugins/openAiPlugin/src/main/resources/editor/vision.json @@ -53,8 +53,9 @@ "subtitle": "A list of messages for Assistant as instructions", "propertyName": "systemMessages", "isRequired": false, - "configProperty": "actionConfiguration.formData.systemMessages", + "configProperty": "actionConfiguration.formData.systemMessages.data", "controlType": "ARRAY_FIELD", + "alternateViewTypes": ["json"], "addMoreButtonLabel": "Add System Message", "customStyles": { "width": "40vw" @@ -75,8 +76,9 @@ "subtitle": "A list of user messages or images. You can pass a link to the image or the base64 encoded image directly in the request.", "propertyName": "userMessages", "isRequired": true, - "configProperty": "actionConfiguration.formData.userMessages", + "configProperty": "actionConfiguration.formData.userMessages.data", "controlType": "ARRAY_FIELD", + "alternateViewTypes": ["json"], "addMoreButtonLabel": "Add User message or Image", "schema": [ { diff --git a/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/ChatCommandTest.java b/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/ChatCommandTest.java index ce8ff5aafa..df4e4372d7 100644 --- a/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/ChatCommandTest.java +++ b/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/ChatCommandTest.java @@ -48,7 +48,7 @@ public class ChatCommandTest { Object messages = List.of( Map.of("role", "user", "content", "Hello"), Map.of("role", "assistant", "content", "Hi there!")); - formData.put("messages", messages); + formData.put("messages", Map.of("data", messages)); ActionConfiguration actionConfiguration = new ActionConfiguration(); actionConfiguration.setFormData(formData); diff --git a/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/VisionCommandTest.java b/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/VisionCommandTest.java index 51389bd81d..f22d27b585 100644 --- a/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/VisionCommandTest.java +++ b/app/server/appsmith-plugins/openAiPlugin/src/test/java/com/external/plugins/VisionCommandTest.java @@ -56,7 +56,8 @@ public class VisionCommandTest { formData.put(MAX_TOKENS, "1000"); formData.put( - SYSTEM_MESSAGES, List.of(Map.of(CONTENT, "Assistant Helper 1"), Map.of(CONTENT, "Assistant Helper 2"))); + SYSTEM_MESSAGES, + Map.of("data", List.of(Map.of(CONTENT, "Assistant Helper 1"), Map.of(CONTENT, "Assistant Helper 2")))); UserQuery userQuery1 = new UserQuery(); userQuery1.setContent("What's in this image?"); @@ -66,7 +67,7 @@ public class VisionCommandTest { userQuery2.setType(QueryType.IMAGE); userQuery2.setContent("https://docs.appsmith.com/img/imagetable.gif"); - formData.put(USER_MESSAGES, List.of(userQuery1, userQuery2)); + formData.put(USER_MESSAGES, Map.of("data", List.of(userQuery1, userQuery2))); ActionConfiguration actionConfiguration = new ActionConfiguration(); actionConfiguration.setFormData(formData); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration039OpenAIMessagesJsToggle.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration039OpenAIMessagesJsToggle.java new file mode 100644 index 0000000000..c515e738bd --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration039OpenAIMessagesJsToggle.java @@ -0,0 +1,95 @@ +package com.appsmith.server.migrations.db.ce; + +import com.appsmith.external.constants.PluginConstants; +import com.appsmith.server.constants.ce.FieldNameCE; +import com.appsmith.server.domains.NewAction; +import com.appsmith.server.domains.Plugin; +import io.mongock.api.annotations.ChangeUnit; +import io.mongock.api.annotations.Execution; +import io.mongock.api.annotations.RollbackExecution; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; +import org.springframework.util.StringUtils; + +import java.util.Map; +import java.util.stream.Stream; + +@Slf4j +@ChangeUnit(order = "039", id = "move-messages-to-data-key-in-openai", author = " ") +public class Migration039OpenAIMessagesJsToggle { + private final MongoTemplate mongoTemplate; + public static final String MESSAGES = "messages"; + public static final String DATA = "data"; + public static final String USER_MESSAGES = "userMessages"; + public static final String SYSTEM_MESSAGES = "systemMessages"; + + public Migration039OpenAIMessagesJsToggle(MongoTemplate mongoTemplate) { + this.mongoTemplate = mongoTemplate; + } + + @RollbackExecution + public void rollbackExecution() {} + + @Execution + public void moveMessagesToDataKeyForSupportingJsToggle() { + // find OpenAI plugin + Plugin plugin = mongoTemplate.findOne( + new Query(Criteria.where(FieldNameCE.PACKAGE_NAME).is(PluginConstants.PackageName.OPEN_AI_PLUGIN)), + Plugin.class); + if (plugin == null || !StringUtils.hasText(plugin.getId())) { + // plugin is not installed, no need of rest of migration steps + log.warn("Unable to find OpenAI plugin in installed datasources"); + return; + } + Query openAiDatasourceQuery = + new Query(Criteria.where(FieldNameCE.PLUGIN_ID).is(plugin.getId())); + // find all actions of OpenAI plugin + Stream actionsStream = mongoTemplate.stream(openAiDatasourceQuery, NewAction.class); + + actionsStream.forEach(action -> { + Query findQuery = new Query(Criteria.where("_id").is(action.getId())); + Update update = new Update(); + boolean anyUpdates = false; + if (action.getUnpublishedAction() != null + && action.getUnpublishedAction().getActionConfiguration() != null + && action.getUnpublishedAction().getActionConfiguration().getFormData() != null) { + Map formData = + action.getUnpublishedAction().getActionConfiguration().getFormData(); + updateFormData(formData); + update.set("unpublishedAction.actionConfiguration.formData", formData); + anyUpdates = true; + } + + if (action.getPublishedAction() != null + && action.getPublishedAction().getActionConfiguration() != null + && action.getPublishedAction().getActionConfiguration().getFormData() != null) { + Map formData = + action.getPublishedAction().getActionConfiguration().getFormData(); + updateFormData(formData); + update.set("publishedAction.actionConfiguration.formData", formData); + anyUpdates = true; + } + if (anyUpdates) { + mongoTemplate.updateFirst(findQuery, update, NewAction.class); + } + }); + } + + /** + * Moves all messages field values into a `data` key so that it becomes compatible to JS Toggle change on messages field in UI + */ + private void updateFormData(Map formData) { + if (formData.containsKey(MESSAGES)) { + formData.put(MESSAGES, Map.of(DATA, formData.get(MESSAGES))); + } + if (formData.containsKey(SYSTEM_MESSAGES)) { + formData.put(SYSTEM_MESSAGES, Map.of(DATA, formData.get(SYSTEM_MESSAGES))); + } + if (formData.containsKey(USER_MESSAGES)) { + formData.put(USER_MESSAGES, Map.of(DATA, formData.get(USER_MESSAGES))); + } + } +}