Merge pull request #1557 from appsmithorg/feature/telemetry

Adding telemetry for community edition

.env.example

@@ -46,4 +46,9 @@ APPSMITH_MAIL_PASSWORD=
 # Email server feature toggles
 # true | false values
 APPSMITH_MAIL_SMTP_AUTH=
-APPSMITH_MAIL_SMTP_TLS_ENABLED=
+APPSMITH_MAIL_SMTP_TLS_ENABLED=
+
+# Disable all telemetry
+# Note: This only takes effect in self-hosted scenarios. 
+# Please visit: https://docs.appsmith.com/telemetry/telemetry to read more about anonymized data collected by Appsmith
+# APPSMITH_DISABLE_TELEMETRY=false

.github/workflows/client.yml

@@ -73,8 +73,10 @@ jobs:
     - name: Run the jest tests
       run: REACT_APP_ENVIRONMENT=${{steps.vars.outputs.REACT_APP_ENVIRONMENT}} yarn run test:unit
 
+    # We burn React environment & the Segment analytics key into the build itself.
+    # This is to ensure that we don't need to configure it in each installation
     - name: Create the bundle
-      run: REACT_APP_ENVIRONMENT=${{steps.vars.outputs.REACT_APP_ENVIRONMENT}} yarn build
+      run: REACT_APP_ENVIRONMENT=${{steps.vars.outputs.REACT_APP_ENVIRONMENT}} REACT_APP_SEGMENT_CE_KEY=${{ secrets.APPSMITH_SEGMENT_CE_KEY }} yarn build
 
     # Upload the build artifact so that it can be used by the test & deploy job in the workflow
     - name: Upload react build bundle

app/client/cypress/setup-test.sh

@@ -34,8 +34,8 @@ sudo docker run --network host --name postgres -d -p 5432:5432 \
  --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 \
  postgres:latest &
 
-echo "Sleeping for 10 seconds to let the servers start"
-sleep 10
+echo "Sleeping for 20 seconds to let the servers start"
+sleep 20
 
 echo "Checking if the containers have started"
 sudo docker ps -a 

app/client/docker/templates/nginx-linux.conf.template

@@ -36,6 +36,7 @@ server {
         sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '${APPSMITH_VERSION_RELEASE_DATE}';
         sub_filter __APPSMITH_INTERCOM_APP_ID__ '${APPSMITH_INTERCOM_APP_ID}';
         sub_filter __APPSMITH_MAIL_ENABLED__ '${APPSMITH_MAIL_ENABLED}';
+        sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}';
     }
 
     location /f {
@@ -103,6 +104,7 @@ server {
         sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '${APPSMITH_VERSION_RELEASE_DATE}';
         sub_filter __APPSMITH_INTERCOM_APP_ID__ '${APPSMITH_INTERCOM_APP_ID}';
         sub_filter __APPSMITH_MAIL_ENABLED__ '${APPSMITH_MAIL_ENABLED}';
+        sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}';
     }
 
     location /f {

app/client/docker/templates/nginx-mac.conf.template

@@ -36,6 +36,7 @@ server {
         sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '${APPSMITH_VERSION_RELEASE_DATE}';
         sub_filter __APPSMITH_INTERCOM_APP_ID__ '${APPSMITH_INTERCOM_APP_ID}';
         sub_filter __APPSMITH_MAIL_ENABLED__ '${APPSMITH_MAIL_ENABLED}';
+        sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}';
     }
 
     location /f {
@@ -104,6 +105,7 @@ server {
         sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '${APPSMITH_VERSION_RELEASE_DATE}';
         sub_filter __APPSMITH_INTERCOM_APP_ID__ '${APPSMITH_INTERCOM_APP_ID}';
         sub_filter __APPSMITH_MAIL_ENABLED__ '${APPSMITH_MAIL_ENABLED}';
+        sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}';
     }
 
 

app/client/package.json

@@ -63,6 +63,7 @@
     "interweave": "^12.1.1",
     "interweave-autolink": "^4.0.1",
     "js-base64": "^3.4.5",
+    "js-sha256": "^0.9.0",
     "json-fn": "^1.1.1",
     "lint-staged": "^9.2.5",
     "localforage": "^1.7.3",
@@ -141,7 +142,6 @@
   "eslintConfig": {
     "extends": "react-app",
     "parser": "@typescript-eslint/parser"
-
   },
   "browserslist": [
     ">0.2%",

app/client/public/index.html

@@ -119,7 +119,10 @@
       enableGoogleOAuth: parseConfig("__APPSMITH_OAUTH2_GOOGLE_CLIENT_ID__").length > 0,
       enableGithubOAuth: parseConfig("__APPSMITH_OAUTH2_GITHUB_CLIENT_ID__").length > 0,
       enableRapidAPI: parseConfig("__APPSMITH_MARKETPLACE_ENABLED__").length > 0,
-      segment: parseConfig("__APPSMITH_SEGMENT_KEY__"),
+      segment: {
+        apiKey: parseConfig("__APPSMITH_SEGMENT_KEY__"),
+        ceKey: parseConfig("__APPSMITH_SEGMENT_CE_KEY__"),
+      },
       optimizely: parseConfig("__APPSMITH_OPTIMIZELY_KEY__"),
       enableMixpanel: parseConfig("__APPSMITH_SEGMENT_KEY__").length > 0,
       algolia: {
@@ -137,6 +140,7 @@
       },
       intercomAppID: parseConfig("__APPSMITH_INTERCOM_APP_ID__"),
       mailEnabled: parseConfig("__APPSMITH_MAIL_ENABLED__").length > 0,
+      disableTelemetry: parseConfig("__APPSMITH_DISABLE_TELEMETRY__").toLowerCase() === 'true' ,
     };
 
   </script>

app/client/src/configs/index.ts

@@ -16,7 +16,10 @@ type INJECTED_CONFIGS = {
   enableGoogleOAuth: boolean;
   enableGithubOAuth: boolean;
   enableRapidAPI: boolean;
-  segment: string;
+  segment: {
+    apiKey: string;
+    ceKey: string;
+  };
   optimizely: string;
   enableMixpanel: boolean;
   google: string;
@@ -34,6 +37,7 @@ type INJECTED_CONFIGS = {
   };
   intercomAppID: string;
   mailEnabled: boolean;
+  disableTelemetry: boolean;
 };
 declare global {
   interface Window {
@@ -66,7 +70,10 @@ const getConfigsFromEnvVars = (): INJECTED_CONFIGS => {
     enableGithubOAuth: process.env.REACT_APP_OAUTH2_GITHUB_CLIENT_ID
       ? process.env.REACT_APP_OAUTH2_GITHUB_CLIENT_ID.length > 0
       : false,
-    segment: process.env.REACT_APP_SEGMENT_KEY || "",
+    segment: {
+      apiKey: process.env.REACT_APP_SEGMENT_KEY || "",
+      ceKey: process.env.REACT_APP_SEGMENT_CE_KEY || "",
+    },
     optimizely: process.env.REACT_APP_OPTIMIZELY_KEY || "",
     enableMixpanel: process.env.REACT_APP_SEGMENT_KEY
       ? process.env.REACT_APP_SEGMENT_KEY.length > 0
@@ -99,6 +106,7 @@ const getConfigsFromEnvVars = (): INJECTED_CONFIGS => {
     mailEnabled: process.env.REACT_APP_MAIL_ENABLED
       ? process.env.REACT_APP_MAIL_ENABLED.length > 0
       : false,
+    disableTelemetry: false,
   };
 };
 
@@ -140,8 +148,8 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => {
     APPSMITH_FEATURE_CONFIGS.sentry.environment,
   );
   const segment = getConfig(
-    ENV_CONFIG.segment,
-    APPSMITH_FEATURE_CONFIGS.segment,
+    ENV_CONFIG.segment.apiKey,
+    APPSMITH_FEATURE_CONFIGS.segment.apiKey,
   );
   const google = getConfig(ENV_CONFIG.google, APPSMITH_FEATURE_CONFIGS.google);
 
@@ -154,7 +162,7 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => {
 
   const algoliaAPIID = getConfig(
     ENV_CONFIG.algolia.apiId,
-    APPSMITH_FEATURE_CONFIGS.algolia.apiKey,
+    APPSMITH_FEATURE_CONFIGS.algolia.apiId,
   );
   const algoliaAPIKey = getConfig(
     ENV_CONFIG.algolia.apiKey,
@@ -165,6 +173,11 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => {
     APPSMITH_FEATURE_CONFIGS.algolia.indexName,
   );
 
+  const segmentCEKey = getConfig(
+    ENV_CONFIG.segment.ceKey,
+    APPSMITH_FEATURE_CONFIGS.segment.ceKey,
+  );
+
   return {
     sentry: {
       enabled: sentryDSN.enabled && sentryRelease.enabled && sentryENV.enabled,
@@ -187,6 +200,7 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => {
     segment: {
       enabled: segment.enabled,
       apiKey: segment.value,
+      ceKey: segmentCEKey.value,
     },
     algolia: {
       enabled: true,
@@ -219,5 +233,6 @@ export const getAppsmithConfigs = (): AppsmithUIConfigs => {
     intercomAppID:
       ENV_CONFIG.intercomAppID || APPSMITH_FEATURE_CONFIGS.intercomAppID,
     mailEnabled: ENV_CONFIG.mailEnabled || APPSMITH_FEATURE_CONFIGS.mailEnabled,
+    disableTelemetry: APPSMITH_FEATURE_CONFIGS.disableTelemetry,
   };
 };

app/client/src/configs/types.ts

@@ -35,6 +35,7 @@ export type AppsmithUIConfigs = {
   segment: {
     enabled: boolean;
     apiKey: string;
+    ceKey: string;
   };
   algolia: {
     enabled: boolean;
@@ -64,4 +65,6 @@ export type AppsmithUIConfigs = {
   };
   intercomAppID: string;
   mailEnabled: boolean;
+
+  disableTelemetry: boolean;
 };

app/client/src/constants/userConstants.ts

@@ -10,6 +10,7 @@ export type User = {
   username: string;
   name: string;
   gender: Gender;
+  anonymousId: string;
 };
 
 export interface UserApplication {
@@ -29,4 +30,5 @@ export const DefaultCurrentUserDetails: User = {
   username: ANONYMOUS_USERNAME,
   applications: [],
   gender: "MALE",
+  anonymousId: "anonymousId",
 };

app/client/src/sagas/userSagas.tsx

@@ -89,7 +89,7 @@ export function* getCurrentUserSaga() {
         !response.data.isAnonymous &&
         response.data.username !== ANONYMOUS_USERNAME
       ) {
-        AnalyticsUtil.identifyUser(response.data.username, response.data);
+        AnalyticsUtil.identifyUser(response.data);
       }
       if (window.location.pathname === BASE_URL) {
         if (response.data.isAnonymous) {

app/client/src/utils/AnalyticsUtil.tsx

@@ -5,6 +5,7 @@ import smartlookClient from "smartlook-client";
 import { getAppsmithConfigs } from "configs";
 import * as Sentry from "@sentry/react";
 import { ANONYMOUS_USERNAME, User } from "../constants/userConstants";
+import { sha256 } from "js-sha256";
 
 export type EventLocation =
   | "LIGHTNING_MENU"
@@ -98,6 +99,8 @@ function getApplicationId(location: Location) {
 }
 
 class AnalyticsUtil {
+  static cachedAnonymoustId: string;
+  static cachedUserId: string;
   static user?: User = undefined;
   static initializeSmartLook(id: string) {
     smartlookClient.init(id);
@@ -169,38 +172,59 @@ class AnalyticsUtil {
     const userData = AnalyticsUtil.user;
     const appId = getApplicationId(windowDoc.location);
     if (userData) {
+      const { segment } = getAppsmithConfigs();
       const app = (userData.applications || []).find(
         (app: any) => app.id === appId,
       );
-      const user = {
-        userId: userData.username,
-        email: userData.email,
-        currentOrgId: userData.currentOrganizationId,
-        appId: appId,
-        appName: app ? app.name : undefined,
-      };
+      let user: any = {};
+      if (segment.enabled && segment.apiKey) {
+        user = {
+          userId: userData.username,
+          email: userData.email,
+          currentOrgId: userData.currentOrganizationId,
+          appId: appId,
+          appName: app ? app.name : undefined,
+        };
+      }
       finalEventData = {
         ...eventData,
         userData: user.userId === ANONYMOUS_USERNAME ? undefined : user,
       };
     }
     if (windowDoc.analytics) {
+      log.debug("Event fired", eventName, finalEventData);
       windowDoc.analytics.track(eventName, finalEventData);
     }
-    log.debug("Event fired", eventName, finalEventData);
   }
 
-  static identifyUser(userId: string, userData: User) {
-    log.debug("Identify User " + userId);
+  static identifyUser(userData: User) {
+    const { segment, smartLook } = getAppsmithConfigs();
     const windowDoc: any = window;
-    AnalyticsUtil.user = userData;
+    const userId = userData.username;
     FeatureFlag.identify(userData);
     if (windowDoc.analytics) {
-      windowDoc.analytics.identify(userId, {
-        email: userData.email,
-        name: userData.name,
-        userId: userId,
-      });
+      // This flag is only set on Appsmith Cloud. In this case, we get more detailed analytics of the user
+      if (segment.apiKey) {
+        const userProperties = {
+          email: userData.email,
+          name: userData.name,
+          userId: userId,
+        };
+        AnalyticsUtil.user = userData;
+        log.debug("Identify User " + userId);
+        windowDoc.analytics.identify(userId, userProperties);
+      } else if (segment.ceKey) {
+        // This is a self-hosted instance. Only send data if the analytics are NOT disabled by the user
+        // This is done by setting environment variable APPSMITH_DISABLE_TELEMETRY in the docker.env file
+        if (userId !== AnalyticsUtil.cachedUserId) {
+          AnalyticsUtil.cachedAnonymoustId = sha256(userId);
+          AnalyticsUtil.cachedUserId = userId;
+        }
+        log.debug(
+          "Identify Anonymous User " + AnalyticsUtil.cachedAnonymoustId,
+        );
+        windowDoc.analytics.identify(AnalyticsUtil.cachedAnonymoustId);
+      }
     }
     Sentry.configureScope(function(scope) {
       scope.setUser({
@@ -209,7 +233,7 @@ class AnalyticsUtil {
         email: userData.email,
       });
     });
-    const { smartLook } = getAppsmithConfigs();
+
     if (smartLook.enabled) {
       smartlookClient.identify(userId, { email: userData.email });
     }

app/client/src/utils/AppsmithUtils.tsx

@@ -48,14 +48,19 @@ export const appInitializer = () => {
   if (appsmithConfigs.sentry.enabled) {
     Sentry.init(appsmithConfigs.sentry);
   }
-  if (appsmithConfigs.smartLook.enabled) {
-    const { id } = appsmithConfigs.smartLook;
-    AnalyticsUtil.initializeSmartLook(id);
+  if (!appsmithConfigs.disableTelemetry) {
+    if (appsmithConfigs.smartLook.enabled) {
+      const { id } = appsmithConfigs.smartLook;
+      AnalyticsUtil.initializeSmartLook(id);
+    }
+    if (appsmithConfigs.segment.enabled && appsmithConfigs.segment.apiKey) {
+      // This value is only enabled for Appsmith's cloud hosted version. It is not set in self-hosted environments
+      AnalyticsUtil.initializeSegment(appsmithConfigs.segment.apiKey);
+    } else if (appsmithConfigs.segment.ceKey) {
+      // This value is set in self-hosted environments. But if the analytics are disabled, it's never used.
+      AnalyticsUtil.initializeSegment(appsmithConfigs.segment.ceKey);
+    }
   }
-  if (appsmithConfigs.segment.enabled) {
-    AnalyticsUtil.initializeSegment(appsmithConfigs.segment.apiKey);
-  }
-
   log.setLevel(getEnvLogLevel(appsmithConfigs.logLevel));
 };
 

app/client/yarn.lock

@@ -11781,6 +11781,11 @@ js-base64@^3.4.5:
   resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-3.5.2.tgz#3cc800e4f10812b55fb5ec53e7cabaef35dc6d3c"
   integrity sha512-VG2qfvV5rEQIVxq9UmAVyWIaOdZGt9M16BLu8vFkyWyhv709Hyg4nKUb5T+Ru+HmAr9RHdF+kQDKAhbJlcdKeQ==
 
+js-sha256@^0.9.0:
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/js-sha256/-/js-sha256-0.9.0.tgz#0b89ac166583e91ef9123644bd3c5334ce9d0966"
+  integrity sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==
+
 js-string-escape@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef"

deploy/install.sh

@@ -706,4 +706,6 @@ else
        }
     }' > /dev/null
 fi
+echo -e "Thank you for installing appsmith! We want to be transparent and inform you that we do perform telemetry in our on-prem installations. This helps us understand your needs and prioritise features & bug fixes better."
+echo -e "All telemetry is 100% anonymous and only statistical in nature. You can read more about it and how to disable it in our documentation https://docs.appsmith.com/telemetry/telemetry"
 echo -e "\nPeace out ✌️\n"

deploy/template/nginx_app.conf.sh

@@ -48,6 +48,7 @@ $NGINX_SSL_CMNT    server_name $custom_domain ;
         sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '\${APPSMITH_VERSION_RELEASE_DATE}';
         sub_filter __APPSMITH_INTERCOM_APP_ID__ '\${APPSMITH_INTERCOM_APP_ID}';
         sub_filter __APPSMITH_MAIL_ENABLED__ '\${APPSMITH_MAIL_ENABLED}';
+        sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}';
     }
 
     location /f {
@@ -103,6 +104,7 @@ $NGINX_SSL_CMNT        sub_filter __APPSMITH_VERSION_ID__ '\${APPSMITH_VERSION_I
 $NGINX_SSL_CMNT        sub_filter __APPSMITH_VERSION_RELEASE_DATE__ '\${APPSMITH_VERSION_RELEASE_DATE}';
 $NGINX_SSL_CMNT        sub_filter __APPSMITH_INTERCOM_APP_ID__ '\${APPSMITH_INTERCOM_APP_ID}';
 $NGINX_SSL_CMNT        sub_filter __APPSMITH_MAIL_ENABLED__ '\${APPSMITH_MAIL_ENABLED}';
+$NGINX_SSL_CMNT        sub_filter __APPSMITH_DISABLE_TELEMETRY__ '${APPSMITH_DISABLE_TELEMETRY}';
 $NGINX_SSL_CMNT    }
 $NGINX_SSL_CMNT
 $NGINX_SSL_CMNT    location /f {