chore: added caching layer for table widget data (#39703)

This commit is contained in:
Aman Agarwal 2025-04-02 14:02:54 +05:30 committed by GitHub
parent 703363f227
commit b8fac1c2ea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 169 additions and 320 deletions

View File

@ -349,7 +349,11 @@ export function Table(props: TableProps) {
{isHeaderVisible && <TableHeader />}
<div className={getTableWrapClassName} ref={tableWrapperRef}>
<div {...getTableProps()} className="table column-freeze">
{shouldUseVirtual ? <VirtualTable /> : <StaticTable />}
{shouldUseVirtual ? (
<VirtualTable ref={scrollBarRef} />
) : (
<StaticTable />
)}
</div>
</div>
</TableWrapper>

View File

@ -99,6 +99,7 @@ const mockTableProviderProps = {
showConnectDataOverlay: false,
onConnectData: jest.fn(),
isInfiniteScrollEnabled: false,
endOfData: false,
};
// Test components
@ -206,6 +207,7 @@ describe("TableContext", () => {
"showConnectDataOverlay",
"onConnectData",
"isInfiniteScrollEnabled",
"endOfData",
].sort();
const result = Object.keys(

View File

@ -147,7 +147,7 @@ const BaseVirtualList = React.memo(function BaseVirtualList({
rowNeedsMeasurement={rowNeedsMeasurement}
/>
);
}, [listRef, rowHeights, rowNeedsMeasurement]);
}, [listRef, rowHeights, rowNeedsMeasurement, hasMoreData]);
return (
<VariableSizeList

View File

@ -1,10 +1,9 @@
import React, { type Ref } from "react";
import React, { useMemo, type Ref } from "react";
import { type ReactElementType } from "react-window";
import type SimpleBar from "simplebar-react";
import { LoadingIndicator } from "../../LoadingIndicator";
import { VariableInfiniteVirtualList } from "../../TableBodyCoreComponents/VirtualList";
import { useAppsmithTable } from "../../TableContext";
import { useInfiniteVirtualization } from "./useInfiniteVirtualization";
interface InfiniteScrollBodyProps {
innerElementType: ReactElementType;
@ -13,6 +12,7 @@ interface InfiniteScrollBodyProps {
const InfiniteScrollBodyComponent = React.forwardRef(
(props: InfiniteScrollBodyProps, ref: Ref<SimpleBar>) => {
const {
endOfData,
height,
isLoading,
nextPageClick,
@ -21,24 +21,24 @@ const InfiniteScrollBodyComponent = React.forwardRef(
tableSizes,
totalRecordsCount,
} = useAppsmithTable();
const { cachedRows, hasMoreData, itemCount } = useInfiniteVirtualization({
rows,
totalRecordsCount,
loadMore: nextPageClick,
pageSize,
});
const itemCount = useMemo(() => {
return totalRecordsCount && totalRecordsCount > 0
? totalRecordsCount
: rows.length;
}, [totalRecordsCount, rows]);
return (
<div className="simplebar-content-wrapper">
<VariableInfiniteVirtualList
hasMoreData={hasMoreData}
hasMoreData={!endOfData}
height={height}
innerElementType={props.innerElementType}
itemCount={itemCount}
loadMore={nextPageClick}
outerRef={ref}
pageSize={pageSize}
rows={cachedRows}
rows={rows}
tableSizes={tableSizes}
/>
{isLoading && <LoadingIndicator />}

View File

@ -1,216 +0,0 @@
import { renderHook } from "@testing-library/react-hooks";
import { useInfiniteVirtualization } from "./useInfiniteVirtualization";
import { act } from "@testing-library/react";
import type { Row as ReactTableRowType } from "react-table";
describe("useInfiniteVirtualization", () => {
// Mock factory function to create test rows
const createMockRows = (
count: number,
startIndex = 0,
): ReactTableRowType<Record<string, unknown>>[] => {
return Array.from({ length: count }, (_, i) => ({
id: `${startIndex + i + 1}`,
original: { id: startIndex + i + 1, name: `Test ${startIndex + i + 1}` },
index: startIndex + i,
cells: [],
values: {},
getRowProps: jest.fn(),
allCells: [],
subRows: [],
isExpanded: false,
canExpand: false,
depth: 0,
toggleRowExpanded: jest.fn(),
state: {},
toggleRowSelected: jest.fn(),
getToggleRowExpandedProps: jest.fn(),
isSelected: false,
isSomeSelected: false,
isGrouped: false,
groupByID: "",
groupByVal: "",
leafRows: [],
getToggleRowSelectedProps: jest.fn(),
setState: jest.fn(),
}));
};
const mockRows = createMockRows(2);
const defaultProps = {
rows: mockRows,
loadMore: jest.fn(),
pageSize: 10,
};
beforeEach(() => {
jest.clearAllMocks();
});
it("1. should return correct itemCount when totalRecordsCount is provided", () => {
const totalRecordsCount = 100;
const { result } = renderHook(() =>
useInfiniteVirtualization({
...defaultProps,
totalRecordsCount,
}),
);
expect(result.current.itemCount).toBe(mockRows.length);
});
it("2. should return rows length as itemCount when totalRecordsCount is not provided", () => {
const { result } = renderHook(() =>
useInfiniteVirtualization(defaultProps),
);
expect(result.current.itemCount).toBe(defaultProps.rows.length);
});
it("3. should update cachedRows when rows are provided", () => {
const { result } = renderHook(() =>
useInfiniteVirtualization(defaultProps),
);
expect(result.current.cachedRows).toEqual(mockRows);
});
it("4. should return zero itemCount when there are no records", () => {
const { result } = renderHook(() =>
useInfiniteVirtualization({
...defaultProps,
rows: [],
}),
);
expect(result.current.itemCount).toBe(0);
});
it("5. should cache rows from multiple page loads", () => {
const pageSize = 2;
const { rerender, result } = renderHook(
(props) => useInfiniteVirtualization(props),
{
initialProps: {
...defaultProps,
rows: createMockRows(pageSize),
pageSize,
},
},
);
// Initial page loaded
expect(result.current.cachedRows.length).toBe(pageSize);
// Load second page
act(() => {
rerender({
...defaultProps,
rows: createMockRows(pageSize, pageSize),
pageSize,
});
});
// Should now have both pages cached
expect(result.current.cachedRows.length).toBe(pageSize * 2);
expect(result.current.cachedRows[0].id).toBe("1");
expect(result.current.cachedRows[2].id).toBe("3");
});
it("6. should handle partial page loads and detect end of data", () => {
const pageSize = 10;
const partialPageSize = 3;
// Initial full page
const { rerender, result } = renderHook(
(props) => useInfiniteVirtualization(props),
{
initialProps: {
...defaultProps,
rows: createMockRows(pageSize),
pageSize,
},
},
);
// Then partial page to signal end of data
act(() => {
rerender({
...defaultProps,
rows: createMockRows(partialPageSize, pageSize),
pageSize,
});
});
// Should mark all items as loaded including those beyond the actual data
expect(result.current.cachedRows.length).toBe(pageSize + partialPageSize);
expect(result.current.itemCount).toBe(pageSize + partialPageSize);
});
it("7. should handle empty page load as end of data", () => {
const pageSize = 5;
// Initial page
const { rerender, result } = renderHook(
(props) => useInfiniteVirtualization(props),
{
initialProps: {
...defaultProps,
rows: createMockRows(pageSize),
pageSize,
},
},
);
// Then empty page to signal end of data
act(() => {
rerender({
...defaultProps,
rows: [],
pageSize,
});
});
// Should identify all items as loaded
expect(result.current.cachedRows.length).toBe(pageSize);
expect(result.current.itemCount).toBe(pageSize);
});
it("8. should maintain correct page reference during multiple rerenders", () => {
const pageSize = 2;
// First page
const { rerender, result } = renderHook(
(props) => useInfiniteVirtualization(props),
{
initialProps: {
...defaultProps,
rows: createMockRows(pageSize),
pageSize,
},
},
);
// Second page
act(() => {
rerender({
...defaultProps,
rows: createMockRows(pageSize, pageSize),
pageSize,
});
});
// Third page
act(() => {
rerender({
...defaultProps,
rows: createMockRows(pageSize, pageSize * 2),
pageSize,
});
});
// Should have all pages cached
expect(result.current.cachedRows.length).toBe(pageSize * 3);
});
});

View File

@ -1,84 +0,0 @@
import { useEffect, useMemo, useRef, useState } from "react";
import type { Row as ReactTableRowType } from "react-table";
export interface UseInfiniteVirtualizationProps {
rows: ReactTableRowType<Record<string, unknown>>[];
totalRecordsCount?: number;
loadMore: () => void;
pageSize: number;
}
export interface UseInfiniteVirtualizationReturn {
itemCount: number;
hasMoreData: boolean;
cachedRows: ReactTableRowType<Record<string, unknown>>[];
}
interface LoadedRowsCache {
[pageIndex: number]: ReactTableRowType<Record<string, unknown>>[];
}
export const useInfiniteVirtualization = ({
loadMore,
pageSize,
rows,
totalRecordsCount,
}: UseInfiniteVirtualizationProps): UseInfiniteVirtualizationReturn => {
const [loadedPages, setLoadedPages] = useState<LoadedRowsCache>({});
const lastLoadedPageRef = useRef<number>(0);
const hasMoreDataRef = useRef<boolean>(true);
useEffect(() => {
if (rows.length > 0) {
const currentPageIndex = lastLoadedPageRef.current;
setLoadedPages((prev) => ({
...prev,
[currentPageIndex]: rows,
}));
// Only increment if we got a full page or some data
if (rows.length === pageSize) {
lastLoadedPageRef.current = currentPageIndex + 1;
} else if (rows.length < pageSize && rows.length > 0) {
// If we got less than a full page, assume this is the last page
hasMoreDataRef.current = false;
}
// load another page in initial load if there is more data to load
if (cachedRows.length < pageSize * 2) {
loadMore();
}
} else if (rows.length === 0 && lastLoadedPageRef.current > 0) {
// If no rows are returned and we've loaded at least one page, assume end of data
hasMoreDataRef.current = false;
}
}, [rows, pageSize]);
const cachedRows = useMemo(() => {
const allRows: ReactTableRowType<Record<string, unknown>>[] = [];
Object.keys(loadedPages)
.map(Number)
.sort((a, b) => a - b)
.forEach((pageIndex) => {
allRows.push(...loadedPages[pageIndex]);
});
return allRows;
}, [loadedPages]);
const itemCount = useMemo(() => {
// If we know there's no more data, cap itemCount at cachedRows.length
if (!hasMoreDataRef.current) {
return cachedRows.length;
}
return cachedRows.length;
}, [totalRecordsCount, cachedRows.length]);
return {
itemCount,
cachedRows,
hasMoreData: hasMoreDataRef.current,
};
};

View File

@ -139,6 +139,7 @@ describe("TableWidget Actions Component", () => {
showConnectDataOverlay: false,
onConnectData: jest.fn(),
isLoading: false,
endOfData: false,
};
const renderWithTableProvider = (props: Partial<TableProviderProps>) => {

View File

@ -108,6 +108,7 @@ interface ReactTableComponentProps {
showConnectDataOverlay: boolean;
onConnectData: () => void;
isInfiniteScrollEnabled: boolean;
endOfData: boolean;
}
function ReactTableComponent(props: ReactTableComponentProps) {
@ -127,6 +128,7 @@ function ReactTableComponent(props: ReactTableComponentProps) {
disableDrag,
editableCell,
editMode,
endOfData,
filters,
handleColumnFreeze,
handleReorderColumn,
@ -243,6 +245,7 @@ function ReactTableComponent(props: ReactTableComponentProps) {
editMode={editMode}
editableCell={editableCell}
enableDrag={memoziedEnableDrag}
endOfData={endOfData}
filters={filters}
handleColumnFreeze={handleColumnFreeze}
handleReorderColumn={handleReorderColumn}

View File

@ -80,6 +80,7 @@ export interface TableProps {
showConnectDataOverlay: boolean;
onConnectData: () => void;
isInfiniteScrollEnabled: boolean;
endOfData: boolean;
}
export interface TableProviderProps extends TableProps {

View File

@ -113,6 +113,8 @@ export interface TableWidgetProps
customIsLoading: boolean;
customIsLoadingValue: boolean;
infiniteScrollEnabled: boolean;
cachedTableData: Record<number, Array<Record<string, unknown>>>;
endOfData: boolean;
}
export enum TableVariantTypes {

View File

@ -108,6 +108,8 @@ describe("TableWidgetV2 getWidgetView", () => {
defaultNewRow: {},
frozenColumnIndices: { a: 1 },
infiniteScrollEnabled: false,
endOfData: false,
cachedTableData: {},
};
describe("TableWidgetV2 loading checks", () => {

View File

@ -0,0 +1,63 @@
import _ from "lodash";
import moment from "moment";
import derivedProperty from "../../derived";
describe("validate getProcessedTableData function", () => {
const defaultInput = {
infiniteScrollEnabled: false,
cachedTableData: {
1: [
{
id: 1,
name: "John",
},
{
id: 2,
name: "Ron",
},
],
2: [
{
id: 3,
name: "Doe",
},
{
id: 4,
name: "Foo",
},
],
},
tableData: [
{
id: 3,
name: "John",
},
{
id: 4,
name: "Ron",
},
],
transientTableData: {},
};
it("should return the tableData as the processData when infiniteScrollEnabled is false", () => {
const { getProcessedTableData } = derivedProperty;
const processedData = getProcessedTableData(defaultInput, moment, _);
expect(processedData.map((i) => i.id)).toStrictEqual([3, 4]);
});
it("should return the cachedTableData as the processData when infiniteScrollEnabled is true", () => {
const { getProcessedTableData } = derivedProperty;
const processedData = getProcessedTableData(
{
...defaultInput,
infiniteScrollEnabled: true,
},
moment,
_,
);
expect(processedData.map((i) => i.id)).toStrictEqual([1, 2, 3, 4]);
});
});

View File

@ -198,10 +198,18 @@ export default {
//
getProcessedTableData: (props, moment, _) => {
let data;
let tableData;
if (_.isArray(props.tableData)) {
if (props.infiniteScrollEnabled) {
/* This logic is needed as the cachedTableData will have data based on each pageNo. Since the object would be { 1: array of page 1 data, 2: array of page 2 data }, hence the values will have array of array data, hence it is flattened to store back in tableData for processing. */
tableData = _.flatten(_.values(props.cachedTableData));
} else {
tableData = props.tableData;
}
if (_.isArray(tableData)) {
/* Populate meta keys (__originalIndex__, __primaryKey__) and transient values */
data = props.tableData.map((row, index) => ({
data = tableData.map((row, index) => ({
...row,
__originalIndex__: index,
__primaryKey__: props.primaryColumnId

View File

@ -229,6 +229,8 @@ class TableWidgetV2 extends BaseWidget<TableWidgetProps, WidgetState> {
: undefined,
customIsLoading: false,
customIsLoadingValue: "",
cachedTableData: {},
endOfData: false,
};
}
@ -912,6 +914,9 @@ class TableWidgetV2 extends BaseWidget<TableWidgetProps, WidgetState> {
//dont neet to batch this since single action
this.hydrateStickyColumns();
}
// Commit Batch Updates property `true` is passed as commitBatchMetaUpdates is not called on componentDidMount and we need to call it for updating the batch updates
this.updateInfiniteScrollProperties(true);
}
componentDidUpdate(prevProps: TableWidgetProps) {
@ -982,18 +987,35 @@ class TableWidgetV2 extends BaseWidget<TableWidgetProps, WidgetState> {
pushBatchMetaUpdates("filters", []);
}
}
/*
* Clear transient table data and editablecell when tableData changes
*/
if (isTableDataModified) {
/*
* Clear transient table data and editablecell when tableData changes
*/
pushBatchMetaUpdates("transientTableData", {});
// reset updatedRowIndex whenever transientTableData is flushed.
pushBatchMetaUpdates("updatedRowIndex", -1);
/*
* Updating the caching layer on table data modification
* Commit Batch Updates property `false` is passed as commitBatchMetaUpdates is called on componentDidUpdate
* and we need not to explicitly call it for updating the batch updates
* */
this.updateInfiniteScrollProperties();
this.pushClearEditableCellsUpdates();
pushBatchMetaUpdates("selectColumnFilterText", {});
} else {
// TODO: reset the widget on any property change, like if the toggle of infinite scroll is enabled and previously it was disabled, currently we update cachedTableData property to the current tableData at pageNo.
/*
* Commit Batch Updates property `false` is passed as commitBatchMetaUpdates is called on componentDidUpdate
* and we need not to explicitly call it for updating the batch updates
* */
if (
!prevProps.infiniteScrollEnabled &&
this.props.infiniteScrollEnabled
) {
this.updateInfiniteScrollProperties();
}
}
if (!pageNo) {
@ -1272,6 +1294,7 @@ class TableWidgetV2 extends BaseWidget<TableWidgetProps, WidgetState> {
disabledAddNewRowSave={this.hasInvalidColumnCell()}
editMode={this.props.renderMode === RenderModes.CANVAS}
editableCell={this.props.editableCell}
endOfData={this.props.endOfData}
filters={this.props.filters}
handleColumnFreeze={this.handleColumnFreeze}
handleReorderColumn={this.handleReorderColumn}
@ -1965,7 +1988,8 @@ class TableWidgetV2 extends BaseWidget<TableWidgetProps, WidgetState> {
*/
if (this.props.isAddRowInProgress) {
row = filteredTableData[rowIndex - 1];
originalIndex = rowIndex === 0 ? -1 : row[ORIGINAL_INDEX_KEY] ?? rowIndex;
originalIndex =
rowIndex === 0 ? -1 : row?.[ORIGINAL_INDEX_KEY] ?? rowIndex;
} else {
row = filteredTableData[rowIndex];
originalIndex = row ? row[ORIGINAL_INDEX_KEY] ?? rowIndex : rowIndex;
@ -2983,6 +3007,45 @@ class TableWidgetV2 extends BaseWidget<TableWidgetProps, WidgetState> {
super.updateOneClickBindingOptionsVisibility(true);
}
};
updateInfiniteScrollProperties(shouldCommitBatchUpdates?: boolean) {
const {
cachedTableData,
commitBatchMetaUpdates,
infiniteScrollEnabled,
pageNo,
pageSize,
processedTableData,
pushBatchMetaUpdates,
tableData,
totalRecordsCount,
} = this.props;
if (infiniteScrollEnabled) {
// Update the cache key for a particular page whenever this function is called. The pageNo data is updated with the tableData.
const updatedCachedTableData = {
...(cachedTableData || {}),
[pageNo]: tableData,
};
pushBatchMetaUpdates("cachedTableData", updatedCachedTableData);
// The check (!!totalRecordsCount && processedTableData.length === totalRecordsCount) is added if the totalRecordsCount property is set then match the length with the processedTableData which has all flatted data from each page in a single array except the current tableData page i.e. [ ...array of page 1 data, ...array of page 2 data ]. Another 'or' check is if (tableData.length < pageSize) when totalRecordsCount is undefined. Table data has a single page data and if the data comes out to be lesser than the pageSize, it is assumed that the data is finished.
if (
(!!totalRecordsCount &&
processedTableData.length + tableData.length === totalRecordsCount) ||
(!totalRecordsCount && tableData.length < pageSize)
) {
pushBatchMetaUpdates("endOfData", true);
} else {
pushBatchMetaUpdates("endOfData", false);
}
if (shouldCommitBatchUpdates) {
commitBatchMetaUpdates();
}
}
}
}
export default TableWidgetV2;