feat: parse CSV data using File Picker Widget (#16689)

* feat: user Papaparser for Parsing csv

* feat: enable dynamic typing

* Update cypress.env.json

* Revert "Update cypress.env.json"

This reverts commit 5ca9f43c8ad90c164b4ac68a9b3764eb9c4bd480.

* fix: reorder property pane

* feat: add helper text for property pane inputs (#16782)

* feat: improve helperText

* feat: add test for csv parsing in filePicker

* fix: proper types

* fix: add more test for helperText

* fix: type error

* fix: props issues

* fix: change file name

* fix: helper text

* fix: test
This commit is contained in:
Tolulope Adetula 2022-09-21 13:59:17 +01:00 committed by GitHub
parent 2088c7e117
commit 42003b97b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 516 additions and 1 deletions

View File

@ -0,0 +1,3 @@
Data Id,String,Number,Boolean,Empty,Date
hsa-miR-942-5p,Blue,23.788,TRUE,,"Wednesday, 20 January 1999"
hsa-miR-943,Black,1000,FALSE,,2022-09-15
1 Data Id String Number Boolean Empty Date
2 hsa-miR-942-5p Blue 23.788 TRUE Wednesday, 20 January 1999
3 hsa-miR-943 Black 1000 FALSE 2022-09-15

View File

@ -0,0 +1,310 @@
{
"dsl": {
"widgetName": "MainContainer",
"backgroundColor": "none",
"rightColumn": 1280,
"snapColumns": 64,
"detachFromLayout": true,
"widgetId": "0",
"topRow": 0,
"bottomRow": 1470,
"containerStyle": "none",
"snapRows": 125,
"parentRowSpace": 1,
"type": "CANVAS_WIDGET",
"canExtend": true,
"version": 61,
"minHeight": 1292,
"parentColumnSpace": 1,
"dynamicTriggerPathList": [],
"dynamicBindingPathList": [],
"leftColumn": 0,
"children": [
{
"boxShadow": "none",
"widgetName": "FilePicker1",
"buttonColor": "{{appsmith.theme.colors.primaryColor}}",
"displayName": "FilePicker",
"iconSVG": "/static/media/icon.7c5ad9c357928c6ff5701bf51a56c2e5.svg",
"searchTags": [
"upload"
],
"topRow": 17,
"bottomRow": 21,
"parentRowSpace": 10,
"allowedFileTypes": [],
"type": "FILE_PICKER_WIDGET_V2",
"hideCard": false,
"animateLoading": true,
"parentColumnSpace": 29.125,
"dynamicTriggerPathList": [],
"leftColumn": 21,
"dynamicBindingPathList": [
{
"key": "buttonColor"
},
{
"key": "borderRadius"
}
],
"isDisabled": false,
"key": "ka3ok4t58q",
"isRequired": false,
"isDeprecated": false,
"rightColumn": 37,
"isDefaultClickDisabled": true,
"widgetId": "tkvh3nxko7",
"isVisible": true,
"label": "Select Files",
"maxFileSize": 5,
"dynamicTyping": true,
"version": 1,
"fileDataType": "Base64",
"parentId": "0",
"selectedFiles": [],
"renderMode": "CANVAS",
"isLoading": false,
"borderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}",
"files": [],
"maxNumFiles": 1
},
{
"boxShadow": "{{appsmith.theme.boxShadow.appBoxShadow}}",
"isVisibleDownload": true,
"iconSVG": "/static/media/icon.db8a9cbd2acd22a31ea91cc37ea2a46c.svg",
"topRow": 40,
"isSortable": true,
"type": "TABLE_WIDGET_V2",
"inlineEditingSaveOption": "ROW_LEVEL",
"animateLoading": true,
"dynamicBindingPathList": [
{
"key": "primaryColumns.step.computedValue"
},
{
"key": "primaryColumns.task.computedValue"
},
{
"key": "primaryColumns.status.computedValue"
},
{
"key": "primaryColumns.action.computedValue"
},
{
"key": "primaryColumns.action.buttonColor"
},
{
"key": "primaryColumns.action.borderRadius"
},
{
"key": "primaryColumns.action.boxShadow"
},
{
"key": "accentColor"
},
{
"key": "borderRadius"
},
{
"key": "boxShadow"
},
{
"key": "childStylesheet.button.buttonColor"
},
{
"key": "childStylesheet.button.borderRadius"
},
{
"key": "childStylesheet.menuButton.menuColor"
},
{
"key": "childStylesheet.menuButton.borderRadius"
},
{
"key": "childStylesheet.iconButton.buttonColor"
},
{
"key": "childStylesheet.iconButton.borderRadius"
},
{
"key": "childStylesheet.editActions.saveButtonColor"
},
{
"key": "childStylesheet.editActions.saveBorderRadius"
},
{
"key": "childStylesheet.editActions.discardButtonColor"
},
{
"key": "childStylesheet.editActions.discardBorderRadius"
},
{
"key": "tableData"
}
],
"leftColumn": 34,
"delimiter": ",",
"defaultSelectedRowIndex": 0,
"accentColor": "{{appsmith.theme.colors.primaryColor}}",
"isVisibleFilters": true,
"isVisible": true,
"enableClientSideSearch": true,
"version": 1,
"totalRecordsCount": 0,
"isLoading": false,
"childStylesheet": {
"button": {
"buttonColor": "{{appsmith.theme.colors.primaryColor}}",
"borderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}",
"boxShadow": "none"
},
"menuButton": {
"menuColor": "{{appsmith.theme.colors.primaryColor}}",
"borderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}",
"boxShadow": "none"
},
"iconButton": {
"buttonColor": "{{appsmith.theme.colors.primaryColor}}",
"borderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}",
"boxShadow": "none"
},
"editActions": {
"saveButtonColor": "{{appsmith.theme.colors.primaryColor}}",
"saveBorderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}",
"discardButtonColor": "{{appsmith.theme.colors.primaryColor}}",
"discardBorderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}"
}
},
"borderRadius": "{{appsmith.theme.borderRadius.appBorderRadius}}",
"defaultSelectedRowIndices": [
0
],
"widgetName": "Table2",
"defaultPageSize": 0,
"columnOrder": [
"step",
"task",
"status",
"action"
],
"dynamicPropertyPathList": [],
"displayName": "Table",
"bottomRow": 72,
"columnWidthMap": {
"task": 245,
"step": 62,
"status": 75
},
"parentRowSpace": 10,
"hideCard": false,
"parentColumnSpace": 29.125,
"dynamicTriggerPathList": [],
"primaryColumns": {
"step": {
"index": 0,
"width": 150,
"id": "step",
"originalId": "step",
"alias": "step",
"horizontalAlignment": "LEFT",
"verticalAlignment": "CENTER",
"columnType": "text",
"textSize": "0.875rem",
"enableFilter": true,
"enableSort": true,
"isVisible": true,
"isCellVisible": true,
"isCellEditable": false,
"isDerived": false,
"label": "step",
"computedValue": "{{Table2.processedTableData.map((currentRow, currentIndex) => ( currentRow[\"step\"]))}}",
"validation": {},
"labelColor": "#FFFFFF"
},
"task": {
"index": 1,
"width": 150,
"id": "task",
"originalId": "task",
"alias": "task",
"horizontalAlignment": "LEFT",
"verticalAlignment": "CENTER",
"columnType": "text",
"textSize": "0.875rem",
"enableFilter": true,
"enableSort": true,
"isVisible": true,
"isCellVisible": true,
"isCellEditable": false,
"isDerived": false,
"label": "task",
"computedValue": "{{Table2.processedTableData.map((currentRow, currentIndex) => ( currentRow[\"task\"]))}}",
"validation": {},
"labelColor": "#FFFFFF"
},
"status": {
"index": 2,
"width": 150,
"id": "status",
"originalId": "status",
"alias": "status",
"horizontalAlignment": "LEFT",
"verticalAlignment": "CENTER",
"columnType": "text",
"textSize": "0.875rem",
"enableFilter": true,
"enableSort": true,
"isVisible": true,
"isCellVisible": true,
"isCellEditable": false,
"isDerived": false,
"label": "status",
"computedValue": "{{Table2.processedTableData.map((currentRow, currentIndex) => ( currentRow[\"status\"]))}}",
"validation": {},
"labelColor": "#FFFFFF"
},
"action": {
"index": 3,
"width": 150,
"id": "action",
"originalId": "action",
"alias": "action",
"horizontalAlignment": "LEFT",
"verticalAlignment": "CENTER",
"columnType": "button",
"textSize": "0.875rem",
"enableFilter": true,
"enableSort": true,
"isVisible": true,
"isCellVisible": true,
"isCellEditable": false,
"isDisabled": false,
"isDerived": false,
"label": "action",
"onClick": "{{currentRow.step === '#1' ? showAlert('Done', 'success') : currentRow.step === '#2' ? navigateTo('https://docs.appsmith.com/core-concepts/connecting-to-data-sources/querying-a-database',undefined,'NEW_WINDOW') : navigateTo('https://docs.appsmith.com/core-concepts/displaying-data-read/display-data-tables',undefined,'NEW_WINDOW')}}",
"computedValue": "{{Table2.processedTableData.map((currentRow, currentIndex) => ( currentRow[\"action\"]))}}",
"validation": {},
"labelColor": "#FFFFFF",
"buttonColor": "{{Table2.processedTableData.map((currentRow, currentIndex) => ( appsmith.theme.colors.primaryColor))}}",
"borderRadius": "{{Table2.processedTableData.map((currentRow, currentIndex) => ( appsmith.theme.borderRadius.appBorderRadius))}}",
"boxShadow": "{{Table2.processedTableData.map((currentRow, currentIndex) => ( 'none'))}}"
}
},
"key": "8azewxqczt",
"isDeprecated": false,
"rightColumn": 64,
"textSize": "0.875rem",
"widgetId": "yyc9usw44s",
"tableData": "{{FilePicker1.files[0].data}}",
"label": "Data",
"searchKey": "",
"parentId": "0",
"renderMode": "CANVAS",
"horizontalAlignment": "LEFT",
"isVisibleSearch": true,
"isVisiblePagination": true,
"verticalAlignment": "CENTER"
}
]
}
}

View File

@ -0,0 +1,51 @@
const commonlocators = require("../../../../../locators/commonlocators.json");
const dsl = require("../../../../../fixtures/filePickerTableDSL.json");
const widgetName = "filepickerwidgetv2";
const ARRAY_CSV_HELPER_TEXT = `All non csv filetypes will have an empty value`;
describe("File picker widget v2", () => {
before(() => {
cy.addDsl(dsl);
});
it("1. Parse CSV data to table Widget", () => {
cy.openPropertyPane(widgetName);
cy.get(
`.t--property-control-dataformat ${commonlocators.helperText}`,
).should("not.exist");
cy.selectDropdownValue(
commonlocators.filePickerDataFormat,
"Array (CSVs only)",
);
cy.get(commonlocators.filePickerDataFormat)
.last()
.should("have.text", "Array (CSVs only)");
cy.get(
`.t--property-control-dataformat ${commonlocators.helperText}`,
).should("exist");
cy.get(
`.t--property-control-dataformat ${commonlocators.helperText}`,
).contains(ARRAY_CSV_HELPER_TEXT);
cy.get(commonlocators.filePickerInput)
.first()
.attachFile("Test_csv.csv");
cy.wait(3000);
cy.readTableV2dataPublish("1", "1").then((tabData) => {
const tabValue = tabData;
expect(tabValue).to.be.equal("Black");
cy.log("the value is" + tabValue);
});
cy.readTableV2dataPublish("1", "2").then((tabData) => {
const tabValue = tabData;
expect(tabValue).to.be.equal("1000");
cy.log("the value is" + tabValue);
});
cy.readTableV2dataPublish("1", "3").then((tabData) => {
const tabValue = tabData;
expect(tabValue).to.be.equal("false");
cy.log("the value is" + tabValue);
});
});
});

View File

@ -125,6 +125,8 @@
"dataType": ".t--property-control-datatype .bp3-popover-target",
"recaptchaVersion": ".t--property-control-googlerecaptchaversion .bp3-popover-target",
"textOverflowDropdown": ".t--property-control-overflowtext .bp3-popover-target",
"filePickerDataFormat": ".t--property-control-dataformat .bp3-popover-target",
"helperText": ".t--property-control-helperText",
"jsonFormFieldType": ".t--property-control-fieldtype .bp3-popover-target",
"jsonFormAddNewCustomFieldBtn": ".t--property-control-fieldconfiguration .t--add-column-btn",
"evaluateMsg": ".t--evaluatedPopup-error",

View File

@ -89,6 +89,7 @@
"nanoid": "^2.0.4",
"node-forge": "^1.3.0",
"normalizr": "^3.3.0",
"papaparse": "^5.3.2",
"path-to-regexp": "^6.2.0",
"popper.js": "^1.15.0",
"prettier": "^1.18.2",
@ -215,6 +216,7 @@
"@types/nanoid": "^2.0.0",
"@types/node": "^10.12.18",
"@types/node-forge": "^0.10.0",
"@types/papaparse": "^5.3.5",
"@types/prismjs": "^1.16.1",
"@types/react": "^16.8.2",
"@types/react-beautiful-dnd": "^11.0.4",

View File

@ -47,7 +47,10 @@ export type PropertyPaneControlConfig = {
id?: string;
label: string | ((props: WidgetProps, propertyPath: string) => string);
propertyName: string;
// Serves in the tooltip
helpText?: string;
//Dynamic text serves below the property pane inputs
helperText?: ((props: WidgetProps) => string) | string;
isJSConvertible?: boolean;
customJSControl?: string;
controlType: ControlType;

View File

@ -50,6 +50,7 @@ import { TooltipComponent } from "design-system";
import { ReactComponent as ResetIcon } from "assets/icons/control/undo_2.svg";
import { AppTheme } from "entities/AppTheming";
import { JS_TOGGLE_DISABLED_MESSAGE } from "@appsmith/constants/messages";
import PropertyPaneHelperText from "./PropertyPaneHelperText";
type Props = PropertyPaneControlConfig & {
panel: IPanelProps;
@ -418,6 +419,9 @@ const PropertyControl = memo((props: Props) => {
const label = isFunction(props.label)
? props.label(widgetProperties, propertyName)
: props.label;
const helperText = isFunction(props.helperText)
? props.helperText(widgetProperties)
: props.helperText;
if (widgetProperties) {
// get the dataTreePath and apply enhancement if exists
@ -618,6 +622,7 @@ const PropertyControl = memo((props: Props) => {
additionAutocomplete,
hideEvaluatedValue(),
)}
<PropertyPaneHelperText helperText={helperText} />
</ControlWrapper>
);
} catch (e) {

View File

@ -0,0 +1,31 @@
import { Text, TextType } from "design-system";
import React from "react";
import { Colors } from "constants/Colors";
import styled from "styled-components";
type Props = {
helperText?: string;
};
const StyledHelperText = styled(Text)`
font-weight: 400;
font-size: 12px;
color: ${Colors.GRAY};
line-height: 14px;
`;
const PropertyPaneHelperText = (props: Props) => {
if (!props.helperText) {
return null;
}
return (
<StyledHelperText
className="t--property-control-helperText"
type={TextType.P1}
>
{props.helperText}
</StyledHelperText>
);
};
export default PropertyPaneHelperText;

View File

@ -18,6 +18,7 @@ export const CONFIG = {
maxNumFiles: 1,
maxFileSize: 5,
fileDataType: FileDataTypes.Base64,
dynamicTyping: true,
widgetName: "FilePicker",
isDefaultClickDisabled: true,
version: 1,

View File

@ -21,6 +21,16 @@ import { createGlobalStyle } from "styled-components";
import UpIcon from "assets/icons/ads/up-arrow.svg";
import CloseIcon from "assets/icons/ads/cross.svg";
import { Colors } from "constants/Colors";
import Papa from "papaparse";
const CSV_ARRAY_LABEL = "Array (CSVs only)";
const CSV_FILE_TYPE_REGEX = /.+(\/csv)$/;
const ARRAY_CSV_HELPER_TEXT = `All non csv filetypes will have an empty value. \n Large files used in widgets directly might slow down the app.`;
const isCSVFileType = (str: string) => CSV_FILE_TYPE_REGEX.test(str);
type Result = string | Buffer | ArrayBuffer | null;
const FilePickerGlobalStyles = createGlobalStyle<{
borderRadius?: string;
@ -317,6 +327,10 @@ class FilePickerWidget extends BaseWidget<
label: FileDataTypes.Text,
value: FileDataTypes.Text,
},
{
label: CSV_ARRAY_LABEL,
value: FileDataTypes.Array,
},
],
isBindProperty: false,
isTriggerProperty: false,
@ -487,6 +501,11 @@ class FilePickerWidget extends BaseWidget<
propertyName: "fileDataType",
label: "Data Format",
controlType: "DROP_DOWN",
helperText: (props: FilePickerWidgetProps) => {
return props.fileDataType === FileDataTypes.Array
? ARRAY_CSV_HELPER_TEXT
: "";
},
options: [
{
label: FileDataTypes.Base64,
@ -500,10 +519,29 @@ class FilePickerWidget extends BaseWidget<
label: FileDataTypes.Text,
value: FileDataTypes.Text,
},
{
label: CSV_ARRAY_LABEL,
value: FileDataTypes.Array,
},
],
isBindProperty: false,
isTriggerProperty: false,
},
{
propertyName: "dynamicTyping",
label: "Infer data-types from CSV",
helpText:
"Controls if the arrays should try to infer the best possible data type based on the values in csv files",
controlType: "SWITCH",
isJSConvertible: false,
isBindProperty: true,
isTriggerProperty: false,
hidden: (props: FilePickerWidgetProps) => {
return props.fileDataType !== FileDataTypes.Array;
},
dependencies: ["fileDataType"],
validation: { type: ValidationTypes.BOOLEAN },
},
{
propertyName: "maxNumFiles",
label: "Max No. of files",
@ -831,7 +869,11 @@ class FilePickerWidget extends BaseWidget<
const newFile = {
type: file.type,
id: file.id,
data: reader.result,
data: this.parseUploadResult(
reader.result,
file.type,
this.props.fileDataType,
),
name: file.meta ? file.meta.name : `File-${index + fileCount}`,
size: file.size,
dataFormat: this.props.fileDataType,
@ -959,6 +1001,57 @@ class FilePickerWidget extends BaseWidget<
);
}
parseUploadResult(
result: Result,
fileType: string,
dataFormat: FileDataTypes,
) {
if (
dataFormat !== FileDataTypes.Array ||
!isCSVFileType(fileType) ||
!result
) {
return result;
}
const data: Record<string, string>[] = [];
const errors: Papa.ParseError[] = [];
function chunk(results: Papa.ParseStepResult<any>) {
if (results?.errors?.length) {
errors.push(...results.errors);
}
data.push(...results.data);
}
if (typeof result === "string") {
const config = {
header: true,
dynamicTyping: this.props.dynamicTyping,
chunk,
};
try {
const startParsing = performance.now();
Papa.parse(result, config);
const endParsing = performance.now();
log.debug(
`### FILE_PICKER_WIDGET_V2 - ${this.props.widgetName} - CSV PARSING `,
`${endParsing - startParsing} ms`,
);
return data;
} catch (error) {
log.error(errors);
return [];
}
} else {
return [];
}
}
static getWidgetType(): WidgetType {
return "FILE_PICKER_WIDGET_V2";
}
@ -981,6 +1074,7 @@ interface FilePickerWidgetProps extends WidgetProps {
backgroundColor: string;
borderRadius: string;
boxShadow?: string;
dynamicTyping?: boolean;
}
export type FilePickerWidgetV2Props = FilePickerWidgetProps;

View File

@ -58,6 +58,7 @@ export enum FileDataTypes {
Base64 = "Base64",
Text = "Text",
Binary = "Binary",
Array = "Array",
}
export type AlignWidget = "LEFT" | "RIGHT";

View File

@ -3155,6 +3155,13 @@
version "2.4.0"
resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz"
"@types/papaparse@^5.3.5":
version "5.3.5"
resolved "https://registry.yarnpkg.com/@types/papaparse/-/papaparse-5.3.5.tgz#e5ad94b1fe98e2a8ea0b03284b83d2cb252bbf39"
integrity sha512-R1icl/hrJPFRpuYj9PVG03WBAlghJj4JW9Py5QdR8FFSxaLmZRyu7xYDCCBZIJNfUv3MYaeBbhBoX958mUTAaw==
dependencies:
"@types/node" "*"
"@types/parse-json@^4.0.0":
version "4.0.0"
resolved "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz"
@ -11242,6 +11249,11 @@ pako@~1.0.2:
resolved "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz"
integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
papaparse@^5.3.2:
version "5.3.2"
resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.3.2.tgz#d1abed498a0ee299f103130a6109720404fbd467"
integrity sha512-6dNZu0Ki+gyV0eBsFKJhYr+MdQYAzFUGlBMNj3GNrmHxmz1lfRa24CjFObPXtjcetlOv5Ad299MhIK0znp3afw==
param-case@^3.0.4:
version "3.0.4"
resolved "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz"