feat: Enable fetch datasource structure for action (#24195)

Users have to toggle the datasource entity to fetch the structure of
their datasources. This PR makes the datasource structure of used
datasources in the current app to be fetched on page load. It also
fetches the datasource structure when a new action is created of a
datasource (that doesn't have its structure present)


Fixes #23958 


- New feature (non-breaking change which adds functionality)

## Testing
>
#### How Has This Been Tested?
> Please describe the tests that you ran to verify your changes. Also
list any relevant details for your test configuration.
> Delete anything that is not relevant
- [ ] Manual
- [ ] Jest
- [ ] Cypress
>
>
#### Test Plan
> Add Testsmith test cases links that relate to this PR
>
>
#### Issues raised during DP testing
> Link issues raised during DP testing for better visiblity and tracking
(copy link from comments dropped on this PR)
>
>
>
## Checklist:
#### Dev activity
- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] PR is being merged under a feature flag


#### QA activity:
- [ ] [Speedbreak
features](https://github.com/appsmithorg/TestSmith/wiki/Test-plan-implementation#speedbreaker-features-to-consider-for-every-change)
have been covered
- [ ] Test plan covers all impacted features and [areas of
interest](https://github.com/appsmithorg/TestSmith/wiki/Guidelines-for-test-plans/_edit#areas-of-interest)
- [ ] Test plan has been peer reviewed by project stakeholders and other
QA members
- [ ] Manually tested functionality on DP
- [ ] We had an implementation alignment call with stakeholders post QA
Round 2
- [ ] Cypress test cases have been added and approved by SDET/manual QA
- [ ] Added `Test Plan Approved` label after Cypress tests were reviewed
- [ ] Added `Test Plan Approved` label after JUnit tests were reviewed

app/client/src/api/ActionAPI.tsx

@@ -40,6 +40,9 @@ export interface QueryConfig {
 export type ActionCreateUpdateResponse = ApiResponse & {
   id: string;
   jsonPathKeys: Record<string, string>;
+  datasource: {
+    id?: string;
+  };
 };
 
 export type PaginationField = "PREV" | "NEXT";

app/client/src/components/editorComponents/WidgetQueryGeneratorForm/CommonControls/TableOrSpreadsheetDropdown/index.tsx

@@ -34,7 +34,9 @@ function TableOrSpreadsheetDropdown() {
           isLoading={isLoading}
           isValid={!error}
           onSelect={(value: string, selectedOption: DefaultOptionType) => {
-            const option = options.find((d) => d.id === selectedOption.key);
+            const option = options.find(
+              (d: DefaultOptionType) => d.id === selectedOption.key,
+            );
 
             if (option) {
               onSelect(value, option);

app/client/src/components/editorComponents/WidgetQueryGeneratorForm/CommonControls/TableOrSpreadsheetDropdown/useTableOrSpreadsheet.tsx

@@ -24,6 +24,7 @@ import type { DropdownOptionType } from "../../types";
 import { getisOneClickBindingConnectingForWidget } from "selectors/oneClickBindingSelectors";
 import AnalyticsUtil from "utils/AnalyticsUtil";
 import { getWidget } from "sagas/selectors";
+import type { DatasourceStructure } from "entities/Datasource";
 
 export function useTableOrSpreadsheet() {
   const dispatch = useDispatch();
@@ -34,8 +35,8 @@ export function useTableOrSpreadsheet() {
 
   const widget = useSelector((state: AppState) => getWidget(state, widgetId));
 
-  const datasourceStructure = useSelector(
-    getDatasourceStructureById(config.datasource),
+  const datasourceStructure: DatasourceStructure = useSelector((state) =>
+    getDatasourceStructureById(state, config.datasource),
   );
 
   const isDatasourceLoading = useSelector(getDatasourceLoading);

app/client/src/sagas/ActionSagas.ts

@@ -9,6 +9,7 @@ import {
 import {
   all,
   call,
+  fork,
   put,
   race,
   select,
@@ -16,7 +17,7 @@ import {
   takeEvery,
   takeLatest,
 } from "redux-saga/effects";
-import type { Datasource } from "entities/Datasource";
+import type { Datasource, DatasourceStructure } from "entities/Datasource";
 import type { ActionCreateUpdateResponse } from "api/ActionAPI";
 import ActionAPI from "api/ActionAPI";
 import type { ApiResponse } from "api/ApiResponses";
@@ -68,6 +69,7 @@ import {
   getActions,
   getCurrentPageNameByActionId,
   getDatasource,
+  getDatasourceStructureById,
   getDatasources,
   getEditorConfig,
   getPageNameByPageId,
@@ -136,6 +138,7 @@ import {
 import { DEFAULT_GRAPHQL_ACTION_CONFIG } from "constants/ApiEditorConstants/GraphQLEditorConstants";
 import { DEFAULT_API_ACTION_CONFIG } from "constants/ApiEditorConstants/ApiEditorConstants";
 import { createNewApiName, createNewQueryName } from "utils/AppsmithUtils";
+import { fetchDatasourceStructure } from "actions/datasourceActions";
 
 export function* createDefaultActionPayload(
   pageId: string,
@@ -256,6 +259,9 @@ export function* createActionSaga(
       const newAction = response.data;
       // @ts-expect-error: type mismatch ActionCreateUpdateResponse vs Action
       yield put(createActionSuccess(newAction));
+
+      // we fork to prevent the call from blocking
+      yield fork(fetchActionDatasourceStructure, newAction);
     }
   } catch (error) {
     yield put({
@@ -265,6 +271,19 @@ export function* createActionSaga(
   }
 }
 
+function* fetchActionDatasourceStructure(action: ActionCreateUpdateResponse) {
+  if (action.datasource?.id) {
+    const doesDatasourceStructureAlreadyExist: DatasourceStructure =
+      yield select(getDatasourceStructureById, action.datasource.id);
+    if (doesDatasourceStructureAlreadyExist) {
+      return;
+    }
+    yield put(fetchDatasourceStructure(action.datasource.id, true));
+  } else {
+    return;
+  }
+}
+
 export function* fetchActionsSaga(
   action: EvaluationReduxAction<FetchActionsPayload>,
 ) {
@@ -632,14 +651,12 @@ function* copyActionSaga(
 
     // checking if there is existing datasource to be added to the action payload
     const existingDatasource = datasources.find(
-      // @ts-expect-error: datasource not present on ActionCreateUpdateResponse
       (d: Datasource) => d.id === response.data.datasource.id,
     );
 
     let payload = response.data;
 
     if (existingDatasource) {
-      // @ts-expect-error: datasource not present on ActionCreateUpdateResponse
       payload = { ...payload, datasource: existingDatasource };
     }
 

app/client/src/sagas/DatasourcesSagas.ts

@@ -2,6 +2,7 @@
 import {
   all,
   call,
+  fork,
   put,
   select,
   take,
@@ -42,6 +43,8 @@ import {
   getPlugin,
   getEditorConfig,
   getPluginByPackageName,
+  getDatasourcesUsedInApplicationByActions,
+  getDatasourceStructureById,
 } from "selectors/entitiesSelector";
 import { addMockDatasourceToWorkspace } from "actions/datasourceActions";
 import type {
@@ -64,6 +67,7 @@ import type { CreateDatasourceConfig } from "api/DatasourcesApi";
 import DatasourcesApi from "api/DatasourcesApi";
 import type {
   Datasource,
+  DatasourceStructure,
   MockDatasource,
   TokenResponse,
 } from "entities/Datasource";
@@ -169,6 +173,32 @@ function* fetchDatasourcesSaga(
   }
 }
 
+function* handleFetchDatasourceStructureOnLoad() {
+  try {
+    // we fork to prevent the call from blocking
+    yield fork(fetchDatasourceStructureOnLoad);
+  } catch (error) {}
+}
+
+function* fetchDatasourceStructureOnLoad() {
+  try {
+    // get datasources of all actions used in the the application
+    const datasourcesUsedInApplication: Datasource[] = yield select(
+      getDatasourcesUsedInApplicationByActions,
+    );
+
+    for (const datasource of datasourcesUsedInApplication) {
+      // it is very unlikely for this to happen, but it does not hurt to check.
+      const doesDatasourceStructureAlreadyExist: DatasourceStructure =
+        yield select(getDatasourceStructureById, datasource.id);
+      if (doesDatasourceStructureAlreadyExist) {
+        continue;
+      }
+      yield put(fetchDatasourceStructure(datasource.id, true));
+    }
+  } catch (error) {}
+}
+
 function* fetchMockDatasourcesSaga() {
   try {
     const response: ApiResponse = yield DatasourcesApi.fetchMockDatasources();
@@ -1798,5 +1828,9 @@ export function* watchDatasourcesSagas() {
       ReduxActionTypes.ADD_AND_FETCH_MOCK_DATASOURCE_STRUCTURE_INIT,
       addAndFetchDatasourceStructureSaga,
     ),
+    takeEvery(
+      ReduxActionTypes.FETCH_DATASOURCES_SUCCESS,
+      handleFetchDatasourceStructureOnLoad,
+    ),
   ]);
 }

app/client/src/selectors/entitiesSelector.ts

@@ -12,6 +12,7 @@ import type {
 } from "entities/Datasource";
 import { isEmbeddedRestDatasource } from "entities/Datasource";
 import type { Action } from "entities/Action";
+import { isStoredDatasource } from "entities/Action";
 import { PluginType } from "entities/Action";
 import { find, get, sortBy } from "lodash";
 import ImageAlt from "assets/images/placeholder-image.svg";
@@ -41,6 +42,7 @@ import recommendedLibraries from "pages/Editor/Explorer/Libraries/recommendedLib
 import type { TJSLibrary } from "workers/common/JSLibrary";
 import { getEntityNameAndPropertyPath } from "@appsmith/workers/Evaluation/evaluationUtils";
 import { getFormValues } from "redux-form";
+import { TEMP_DATASOURCE_ID } from "constants/Datasource";
 
 export const getEntities = (state: AppState): AppState["entities"] =>
   state.entities;
@@ -59,15 +61,16 @@ export const getDatasourcesStructure = (
   return state.entities.datasources.structure;
 };
 
-export const getDatasourceStructureById =
-  (id: string) =>
-  (state: AppState): DatasourceStructure => {
-    return state.entities.datasources.structure[id];
-  };
+export const getDatasourceStructureById = (
+  state: AppState,
+  id: string,
+): DatasourceStructure => {
+  return state.entities.datasources.structure[id];
+};
 
 export const getDatasourceTableColumns =
   (datasourceId: string, tableName: string) => (state: AppState) => {
-    const structure = getDatasourceStructureById(datasourceId)(state);
+    const structure = getDatasourceStructureById(state, datasourceId);
 
     if (structure) {
       const table = structure.tables?.find((d) => d.name === tableName);
@@ -77,7 +80,7 @@ export const getDatasourceTableColumns =
   };
 export const getDatasourceTablePrimaryColumn =
   (datasourceId: string, tableName: string) => (state: AppState) => {
-    const structure = getDatasourceStructureById(datasourceId)(state);
+    const structure = getDatasourceStructureById(state, datasourceId);
 
     if (structure) {
       const table = structure.tables?.find((d) => d.name === tableName);
@@ -1086,3 +1089,27 @@ export const getDatasourceScopeValue = (
   )?.label;
   return label;
 };
+
+export const getDatasourcesUsedInApplicationByActions = (
+  state: AppState,
+): Datasource[] => {
+  const actions = getActions(state);
+  const datasources = getDatasources(state);
+  const datasourceIdsUsedInCurrentApplication = actions.reduce(
+    (acc, action: ActionData) => {
+      if (
+        isStoredDatasource(action.config.datasource) &&
+        action.config.datasource.id
+      ) {
+        acc.add(action.config.datasource.id);
+      }
+      return acc;
+    },
+    new Set(),
+  );
+  return datasources.filter(
+    (ds) =>
+      datasourceIdsUsedInCurrentApplication.has(ds.id) &&
+      ds.id !== TEMP_DATASOURCE_ID,
+  );
+};