diff --git a/.env.example b/.env.example index 2eb1d2231f..a25d8a64d5 100644 --- a/.env.example +++ b/.env.example @@ -46,4 +46,9 @@ APPSMITH_MAIL_PASSWORD= # Email server feature toggles # true | false values APPSMITH_MAIL_SMTP_AUTH= -APPSMITH_MAIL_SMTP_TLS_ENABLED= \ No newline at end of file +APPSMITH_MAIL_SMTP_TLS_ENABLED= + +# Disable all telemetry +# Note: This only takes effect in self-hosted scenarios. +# Please visit: https://docs.appsmith.com/telemetry/telemetry to read more about anonymized data collected by Appsmith +# APPSMITH_DISABLE_TELEMETRY=false \ No newline at end of file diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index e0be8f18b9..4d9ac96cad 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -73,8 +73,10 @@ jobs: - name: Run the jest tests run: REACT_APP_ENVIRONMENT=${{steps.vars.outputs.REACT_APP_ENVIRONMENT}} yarn run test:unit + # We burn React environment & the Segment analytics key into the build itself. + # This is to ensure that we don't need to configure it in each installation - name: Create the bundle - run: REACT_APP_ENVIRONMENT=${{steps.vars.outputs.REACT_APP_ENVIRONMENT}} yarn build + run: REACT_APP_ENVIRONMENT=${{steps.vars.outputs.REACT_APP_ENVIRONMENT}} REACT_APP_SEGMENT_CE_KEY=${{ secrets.APPSMITH_SEGMENT_CE_KEY }} yarn build # Upload the build artifact so that it can be used by the test & deploy job in the workflow - name: Upload react build bundle diff --git a/app/client/cypress/setup-test.sh b/app/client/cypress/setup-test.sh index 8aa8e73f82..944f88aa74 100755 --- a/app/client/cypress/setup-test.sh +++ b/app/client/cypress/setup-test.sh @@ -34,8 +34,8 @@ sudo docker run --network host --name postgres -d -p 5432:5432 \ --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 \ postgres:latest & -echo "Sleeping for 10 seconds to let the servers start" -sleep 10 +echo "Sleeping for 20 seconds to let the servers start" +sleep 20 echo "Checking if the containers have started" sudo docker ps -a diff --git a/app/client/docker/templates/nginx-linux.conf.template b/app/client/docker/templates/nginx-linux.conf.template index b5e37d7935..d205da5d6f 100644 --- a/app/client/docker/templates/nginx-linux.conf.template +++ b/app/client/docker/templates/nginx-linux.conf.template @@ -36,6 +36,7 @@ server { sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '${APPSMITH_VERSION_RELEASE_DATE}'; sub_filter __APPSMITH_INTERCOM_APP_ID__ '${APPSMITH_INTERCOM_APP_ID}'; sub_filter __APPSMITH_MAIL_ENABLED__ '${APPSMITH_MAIL_ENABLED}'; + sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}'; } location /f { @@ -103,6 +104,7 @@ server { sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '${APPSMITH_VERSION_RELEASE_DATE}'; sub_filter __APPSMITH_INTERCOM_APP_ID__ '${APPSMITH_INTERCOM_APP_ID}'; sub_filter __APPSMITH_MAIL_ENABLED__ '${APPSMITH_MAIL_ENABLED}'; + sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}'; } location /f { diff --git a/app/client/docker/templates/nginx-mac.conf.template b/app/client/docker/templates/nginx-mac.conf.template index e36919c4e8..16c7a250e7 100644 --- a/app/client/docker/templates/nginx-mac.conf.template +++ b/app/client/docker/templates/nginx-mac.conf.template @@ -36,6 +36,7 @@ server { sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '${APPSMITH_VERSION_RELEASE_DATE}'; sub_filter __APPSMITH_INTERCOM_APP_ID__ '${APPSMITH_INTERCOM_APP_ID}'; sub_filter __APPSMITH_MAIL_ENABLED__ '${APPSMITH_MAIL_ENABLED}'; + sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}'; } location /f { @@ -104,6 +105,7 @@ server { sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '${APPSMITH_VERSION_RELEASE_DATE}'; sub_filter __APPSMITH_INTERCOM_APP_ID__ '${APPSMITH_INTERCOM_APP_ID}'; sub_filter __APPSMITH_MAIL_ENABLED__ '${APPSMITH_MAIL_ENABLED}'; + sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}'; } diff --git a/app/client/package.json b/app/client/package.json index 61b6ece608..d93d2d07ca 100644 --- a/app/client/package.json +++ b/app/client/package.json @@ -63,6 +63,7 @@ "interweave": "^12.1.1", "interweave-autolink": "^4.0.1", "js-base64": "^3.4.5", + "js-sha256": "^0.9.0", "json-fn": "^1.1.1", "lint-staged": "^9.2.5", "localforage": "^1.7.3", @@ -141,7 +142,6 @@ "eslintConfig": { "extends": "react-app", "parser": "@typescript-eslint/parser" - }, "browserslist": [ ">0.2%", diff --git a/app/client/public/index.html b/app/client/public/index.html index 5178386def..bc1bbbd50c 100755 --- a/app/client/public/index.html +++ b/app/client/public/index.html @@ -119,7 +119,10 @@ enableGoogleOAuth: parseConfig("__APPSMITH_OAUTH2_GOOGLE_CLIENT_ID__").length > 0, enableGithubOAuth: parseConfig("__APPSMITH_OAUTH2_GITHUB_CLIENT_ID__").length > 0, enableRapidAPI: parseConfig("__APPSMITH_MARKETPLACE_ENABLED__").length > 0, - segment: parseConfig("__APPSMITH_SEGMENT_KEY__"), + segment: { + apiKey: parseConfig("__APPSMITH_SEGMENT_KEY__"), + ceKey: parseConfig("__APPSMITH_SEGMENT_CE_KEY__"), + }, optimizely: parseConfig("__APPSMITH_OPTIMIZELY_KEY__"), enableMixpanel: parseConfig("__APPSMITH_SEGMENT_KEY__").length > 0, algolia: { @@ -137,6 +140,7 @@ }, intercomAppID: parseConfig("__APPSMITH_INTERCOM_APP_ID__"), mailEnabled: parseConfig("__APPSMITH_MAIL_ENABLED__").length > 0, + disableTelemetry: parseConfig("__APPSMITH_DISABLE_TELEMETRY__").toLowerCase() === 'true' , }; diff --git a/app/client/src/configs/index.ts b/app/client/src/configs/index.ts index ae41216bb8..d7c734bffe 100644 --- a/app/client/src/configs/index.ts +++ b/app/client/src/configs/index.ts @@ -16,7 +16,10 @@ type INJECTED_CONFIGS = { enableGoogleOAuth: boolean; enableGithubOAuth: boolean; enableRapidAPI: boolean; - segment: string; + segment: { + apiKey: string; + ceKey: string; + }; optimizely: string; enableMixpanel: boolean; google: string; @@ -34,6 +37,7 @@ type INJECTED_CONFIGS = { }; intercomAppID: string; mailEnabled: boolean; + disableTelemetry: boolean; }; declare global { interface Window { @@ -66,7 +70,10 @@ const getConfigsFromEnvVars = (): INJECTED_CONFIGS => { enableGithubOAuth: process.env.REACT_APP_OAUTH2_GITHUB_CLIENT_ID ? process.env.REACT_APP_OAUTH2_GITHUB_CLIENT_ID.length > 0 : false, - segment: process.env.REACT_APP_SEGMENT_KEY || "", + segment: { + apiKey: process.env.REACT_APP_SEGMENT_KEY || "", + ceKey: process.env.REACT_APP_SEGMENT_CE_KEY || "", + }, optimizely: process.env.REACT_APP_OPTIMIZELY_KEY || "", enableMixpanel: process.env.REACT_APP_SEGMENT_KEY ? process.env.REACT_APP_SEGMENT_KEY.length > 0 @@ -99,6 +106,7 @@ const getConfigsFromEnvVars = (): INJECTED_CONFIGS => { mailEnabled: process.env.REACT_APP_MAIL_ENABLED ? process.env.REACT_APP_MAIL_ENABLED.length > 0 : false, + disableTelemetry: false, }; }; @@ -140,8 +148,8 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => { APPSMITH_FEATURE_CONFIGS.sentry.environment, ); const segment = getConfig( - ENV_CONFIG.segment, - APPSMITH_FEATURE_CONFIGS.segment, + ENV_CONFIG.segment.apiKey, + APPSMITH_FEATURE_CONFIGS.segment.apiKey, ); const google = getConfig(ENV_CONFIG.google, APPSMITH_FEATURE_CONFIGS.google); @@ -154,7 +162,7 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => { const algoliaAPIID = getConfig( ENV_CONFIG.algolia.apiId, - APPSMITH_FEATURE_CONFIGS.algolia.apiKey, + APPSMITH_FEATURE_CONFIGS.algolia.apiId, ); const algoliaAPIKey = getConfig( ENV_CONFIG.algolia.apiKey, @@ -165,6 +173,11 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => { APPSMITH_FEATURE_CONFIGS.algolia.indexName, ); + const segmentCEKey = getConfig( + ENV_CONFIG.segment.ceKey, + APPSMITH_FEATURE_CONFIGS.segment.ceKey, + ); + return { sentry: { enabled: sentryDSN.enabled && sentryRelease.enabled && sentryENV.enabled, @@ -187,6 +200,7 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => { segment: { enabled: segment.enabled, apiKey: segment.value, + ceKey: segmentCEKey.value, }, algolia: { enabled: true, @@ -219,5 +233,6 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => { intercomAppID: ENV_CONFIG.intercomAppID || APPSMITH_FEATURE_CONFIGS.intercomAppID, mailEnabled: ENV_CONFIG.mailEnabled || APPSMITH_FEATURE_CONFIGS.mailEnabled, + disableTelemetry: APPSMITH_FEATURE_CONFIGS.disableTelemetry, }; }; diff --git a/app/client/src/configs/types.ts b/app/client/src/configs/types.ts index 34a16b9cfd..7007f200fd 100644 --- a/app/client/src/configs/types.ts +++ b/app/client/src/configs/types.ts @@ -35,6 +35,7 @@ export type AppsmithUIConfigs = { segment: { enabled: boolean; apiKey: string; + ceKey: string; }; algolia: { enabled: boolean; @@ -64,4 +65,6 @@ export type AppsmithUIConfigs = { }; intercomAppID: string; mailEnabled: boolean; + + disableTelemetry: boolean; }; diff --git a/app/client/src/constants/userConstants.ts b/app/client/src/constants/userConstants.ts index 16bbb45ac3..17e69835a1 100644 --- a/app/client/src/constants/userConstants.ts +++ b/app/client/src/constants/userConstants.ts @@ -10,6 +10,7 @@ export type User = { username: string; name: string; gender: Gender; + anonymousId: string; }; export interface UserApplication { @@ -29,4 +30,5 @@ export const DefaultCurrentUserDetails: User = { username: ANONYMOUS_USERNAME, applications: [], gender: "MALE", + anonymousId: "anonymousId", }; diff --git a/app/client/src/sagas/userSagas.tsx b/app/client/src/sagas/userSagas.tsx index 8a37087b9c..cbb1916ebe 100644 --- a/app/client/src/sagas/userSagas.tsx +++ b/app/client/src/sagas/userSagas.tsx @@ -89,7 +89,7 @@ export function* getCurrentUserSaga() { !response.data.isAnonymous && response.data.username !== ANONYMOUS_USERNAME ) { - AnalyticsUtil.identifyUser(response.data.username, response.data); + AnalyticsUtil.identifyUser(response.data); } if (window.location.pathname === BASE_URL) { if (response.data.isAnonymous) { diff --git a/app/client/src/utils/AnalyticsUtil.tsx b/app/client/src/utils/AnalyticsUtil.tsx index 21e1aab6c3..09600ce41d 100644 --- a/app/client/src/utils/AnalyticsUtil.tsx +++ b/app/client/src/utils/AnalyticsUtil.tsx @@ -5,6 +5,7 @@ import smartlookClient from "smartlook-client"; import { getAppsmithConfigs } from "configs"; import * as Sentry from "@sentry/react"; import { ANONYMOUS_USERNAME, User } from "../constants/userConstants"; +import { sha256 } from "js-sha256"; export type EventLocation = | "LIGHTNING_MENU" @@ -98,6 +99,8 @@ function getApplicationId(location: Location) { } class AnalyticsUtil { + static cachedAnonymoustId: string; + static cachedUserId: string; static user?: User = undefined; static initializeSmartLook(id: string) { smartlookClient.init(id); @@ -169,38 +172,59 @@ class AnalyticsUtil { const userData = AnalyticsUtil.user; const appId = getApplicationId(windowDoc.location); if (userData) { + const { segment } = getAppsmithConfigs(); const app = (userData.applications || []).find( (app: any) => app.id === appId, ); - const user = { - userId: userData.username, - email: userData.email, - currentOrgId: userData.currentOrganizationId, - appId: appId, - appName: app ? app.name : undefined, - }; + let user: any = {}; + if (segment.enabled && segment.apiKey) { + user = { + userId: userData.username, + email: userData.email, + currentOrgId: userData.currentOrganizationId, + appId: appId, + appName: app ? app.name : undefined, + }; + } finalEventData = { ...eventData, userData: user.userId === ANONYMOUS_USERNAME ? undefined : user, }; } if (windowDoc.analytics) { + log.debug("Event fired", eventName, finalEventData); windowDoc.analytics.track(eventName, finalEventData); } - log.debug("Event fired", eventName, finalEventData); } - static identifyUser(userId: string, userData: User) { - log.debug("Identify User " + userId); + static identifyUser(userData: User) { + const { segment, smartLook } = getAppsmithConfigs(); const windowDoc: any = window; - AnalyticsUtil.user = userData; + const userId = userData.username; FeatureFlag.identify(userData); if (windowDoc.analytics) { - windowDoc.analytics.identify(userId, { - email: userData.email, - name: userData.name, - userId: userId, - }); + // This flag is only set on Appsmith Cloud. In this case, we get more detailed analytics of the user + if (segment.apiKey) { + const userProperties = { + email: userData.email, + name: userData.name, + userId: userId, + }; + AnalyticsUtil.user = userData; + log.debug("Identify User " + userId); + windowDoc.analytics.identify(userId, userProperties); + } else if (segment.ceKey) { + // This is a self-hosted instance. Only send data if the analytics are NOT disabled by the user + // This is done by setting environment variable APPSMITH_DISABLE_TELEMETRY in the docker.env file + if (userId !== AnalyticsUtil.cachedUserId) { + AnalyticsUtil.cachedAnonymoustId = sha256(userId); + AnalyticsUtil.cachedUserId = userId; + } + log.debug( + "Identify Anonymous User " + AnalyticsUtil.cachedAnonymoustId, + ); + windowDoc.analytics.identify(AnalyticsUtil.cachedAnonymoustId); + } } Sentry.configureScope(function(scope) { scope.setUser({ @@ -209,7 +233,7 @@ class AnalyticsUtil { email: userData.email, }); }); - const { smartLook } = getAppsmithConfigs(); + if (smartLook.enabled) { smartlookClient.identify(userId, { email: userData.email }); } diff --git a/app/client/src/utils/AppsmithUtils.tsx b/app/client/src/utils/AppsmithUtils.tsx index e1bd392e19..6d4a75e108 100644 --- a/app/client/src/utils/AppsmithUtils.tsx +++ b/app/client/src/utils/AppsmithUtils.tsx @@ -48,14 +48,19 @@ export const appInitializer = () => { if (appsmithConfigs.sentry.enabled) { Sentry.init(appsmithConfigs.sentry); } - if (appsmithConfigs.smartLook.enabled) { - const { id } = appsmithConfigs.smartLook; - AnalyticsUtil.initializeSmartLook(id); + if (!appsmithConfigs.disableTelemetry) { + if (appsmithConfigs.smartLook.enabled) { + const { id } = appsmithConfigs.smartLook; + AnalyticsUtil.initializeSmartLook(id); + } + if (appsmithConfigs.segment.enabled && appsmithConfigs.segment.apiKey) { + // This value is only enabled for Appsmith's cloud hosted version. It is not set in self-hosted environments + AnalyticsUtil.initializeSegment(appsmithConfigs.segment.apiKey); + } else if (appsmithConfigs.segment.ceKey) { + // This value is set in self-hosted environments. But if the analytics are disabled, it's never used. + AnalyticsUtil.initializeSegment(appsmithConfigs.segment.ceKey); + } } - if (appsmithConfigs.segment.enabled) { - AnalyticsUtil.initializeSegment(appsmithConfigs.segment.apiKey); - } - log.setLevel(getEnvLogLevel(appsmithConfigs.logLevel)); }; diff --git a/app/client/yarn.lock b/app/client/yarn.lock index fc89847cbf..e0f7dac7b5 100644 --- a/app/client/yarn.lock +++ b/app/client/yarn.lock @@ -11781,6 +11781,11 @@ js-base64@^3.4.5: resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-3.5.2.tgz#3cc800e4f10812b55fb5ec53e7cabaef35dc6d3c" integrity sha512-VG2qfvV5rEQIVxq9UmAVyWIaOdZGt9M16BLu8vFkyWyhv709Hyg4nKUb5T+Ru+HmAr9RHdF+kQDKAhbJlcdKeQ== +js-sha256@^0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/js-sha256/-/js-sha256-0.9.0.tgz#0b89ac166583e91ef9123644bd3c5334ce9d0966" + integrity sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA== + js-string-escape@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" diff --git a/deploy/install.sh b/deploy/install.sh index 194ebe7386..0602c48efe 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -706,4 +706,6 @@ else } }' > /dev/null fi +echo -e "Thank you for installing appsmith! We want to be transparent and inform you that we do perform telemetry in our on-prem installations. This helps us understand your needs and prioritise features & bug fixes better." +echo -e "All telemetry is 100% anonymous and only statistical in nature. You can read more about it and how to disable it in our documentation https://docs.appsmith.com/telemetry/telemetry" echo -e "\nPeace out ✌️\n" diff --git a/deploy/template/nginx_app.conf.sh b/deploy/template/nginx_app.conf.sh index 14c27a9eeb..f116fa3028 100644 --- a/deploy/template/nginx_app.conf.sh +++ b/deploy/template/nginx_app.conf.sh @@ -48,6 +48,7 @@ $NGINX_SSL_CMNT server_name $custom_domain ; sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '\${APPSMITH_VERSION_RELEASE_DATE}'; sub_filter __APPSMITH_INTERCOM_APP_ID__ '\${APPSMITH_INTERCOM_APP_ID}'; sub_filter __APPSMITH_MAIL_ENABLED__ '\${APPSMITH_MAIL_ENABLED}'; + sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}'; } location /f { @@ -103,6 +104,7 @@ $NGINX_SSL_CMNT sub_filter __APPSMITH_VERSION_ID__ '\${APPSMITH_VERSION_I $NGINX_SSL_CMNT sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '\${APPSMITH_VERSION_RELEASE_DATE}'; $NGINX_SSL_CMNT sub_filter __APPSMITH_INTERCOM_APP_ID__ '\${APPSMITH_INTERCOM_APP_ID}'; $NGINX_SSL_CMNT sub_filter __APPSMITH_MAIL_ENABLED__ '\${APPSMITH_MAIL_ENABLED}'; +$NGINX_SSL_CMNT sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}'; $NGINX_SSL_CMNT } $NGINX_SSL_CMNT $NGINX_SSL_CMNT location /f {