From 4c2cd986833c0f4f8b1577919053ade5bfe456fa Mon Sep 17 00:00:00 2001 From: balajisoundar Date: Tue, 10 Jan 2023 17:39:33 +0530 Subject: [PATCH] chore: Use rolling window in the usage pulse call and send segment anonymous id for anonymous users (#19552) ## Description We have updated the user tracking to use a rolling window. Now the session starts when the user goes to the builder or viewer for the first time and the subsequent activity tracking will be checked only after an hour. For anonymous users, we send the Segment anonymous id in the usage calls. When the telemetry is off, we still initiate the segment, get the id and then purge the analytics global object. Fixed window (in release, as of now)- if the user starts a session at 01:15 pm, we take 01:00 pm as the session start time and we will check for the next user activity at 2:00 pm. rolling window (in this PR) - if the user starts a session at 01:15 pm, we take 01:15 pm as the session start time and we will check for the next user activity at 2:15 pm. Fixes https://github.com/appsmithorg/cloud-services/issues/183 ## Type of change - New feature (non-breaking change which adds functionality) ## How Has This Been Tested? - Manual ### Test Plan ### Issues raised during DP testing > Link issues raised during DP testing for better visiblity and tracking (copy link from comments dropped on this PR) ## Checklist: ### Dev activity - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] PR is being merged under a feature flag ### QA activity: - [ ] Test plan has been approved by relevant developers - [ ] Test plan has been peer reviewed by QA - [ ] Cypress test cases have been added and approved by either SDET or manual QA - [ ] Organized project review call with relevant stakeholders after Round 1/2 of QA - [ ] Added Test Plan Approved label after reveiwing all Cypress test --- app/client/public/index.html | 1 - app/client/public/logger.js | 93 --------------- app/client/src/RouteBuilder.ts | 7 +- app/client/src/ce/sagas/userSagas.tsx | 40 ++++++- app/client/src/constants/routes/appRoutes.ts | 18 ++- app/client/src/index.tsx | 1 + app/client/src/usagePulse/index.ts | 118 +++++++++++++++++++ app/client/src/usagePulse/usagePulse.test.ts | 32 +++++ app/client/src/utils/AnalyticsUtil.tsx | 17 +++ app/client/src/utils/AppsmithUtils.tsx | 18 +++ 10 files changed, 242 insertions(+), 103 deletions(-) delete mode 100644 app/client/public/logger.js create mode 100644 app/client/src/usagePulse/index.ts create mode 100644 app/client/src/usagePulse/usagePulse.test.ts diff --git a/app/client/public/index.html b/app/client/public/index.html index 356c1c9019..2ca3be33cb 100755 --- a/app/client/public/index.html +++ b/app/client/public/index.html @@ -46,7 +46,6 @@ } - diff --git a/app/client/public/logger.js b/app/client/public/logger.js deleted file mode 100644 index 13b24c245b..0000000000 --- a/app/client/public/logger.js +++ /dev/null @@ -1,93 +0,0 @@ -// Gets the current Unix timestamp -function getCurrentUTCTimestamp(date) { - return Math.floor((date || new Date()).getTime() / 1000); -} - -// Gets the unix timestamp of the hour -// For a current time of 19:15, returns the timestamp of 19:00 -function getCurrentUTCHourTimestamp() { - const date = new Date(); - date.setUTCMinutes(0); - date.setUTCSeconds(0); - date.setUTCMilliseconds(0); - return getCurrentUTCTimestamp(date); -} - -const PULSE_API_ENDPOINT = "/api/v1/usage-pulse"; - -/** - * Sends HTTP pulse to the server, when beaconAPI is not available. - * Fire and forget. - */ -function sendHTTPPulse() { - fetch(PULSE_API_ENDPOINT, { - method: "POST", - credentials: "same-origin", - }) - .then(() => { - // Fire and forget - }) - .catch(() => { - // Ignore errors; fire and forget - }); -} - -/** - * Sends a usage-pulse to the server using the Beacon API. - * If the Beacon API is not available, falls back to a standard fetch. - * Note: Only sends pulse when user is on "/app/" pages: editor and viewer. - */ -function sendPulse() { - if (window.location.href.includes("/app/")) { - navigator.sendBeacon(PULSE_API_ENDPOINT, "") || sendHTTPPulse(); - } -} - -// Checks if the it is time to send another pulse -function shouldSendPulse() { - const timestamp = getCurrentUTCTimestamp(); - return NEXT_LOGGING_HOUR < timestamp; -} - -function addActivityListener() { - window.document.body.addEventListener("pointerdown", punchIn); -} -function removeActivityListener() { - window.document.body.removeEventListener("pointerdown", punchIn); -} - -// Removes event listeners and adds them just in time for the next pulse -function scheduleNextPunchIn() { - const timestamp = getCurrentUTCTimestamp(); - const startListentingIn = NEXT_LOGGING_HOUR - timestamp - 2; - - // If we don't have much time until TTL expires; - // Don't bother removing listener - if (startListentingIn <= 10) return; - - // Remove all listeners for now. - removeActivityListener(); - - // Add listeners 2 seconds before the next hour begins - setTimeout(addActivityListener, startListentingIn * 1000); -} - -LAST_LOGGED_HOUR = 0; // The last time we logged -NEXT_LOGGING_HOUR = 0; // The next time we should log - -function punchIn() { - if (!LAST_LOGGED_HOUR) { - // When this is the first time we're logging - LAST_LOGGED_HOUR = getCurrentUTCHourTimestamp(); - NEXT_LOGGING_HOUR = LAST_LOGGED_HOUR + 3600; - } else { - // Make sure it is time to send the pulse again - if (!shouldSendPulse) return; - LAST_LOGGED_HOUR = NEXT_LOGGING_HOUR; - NEXT_LOGGING_HOUR = LAST_LOGGED_HOUR + 3600; - } - sendPulse(); - scheduleNextPunchIn(); -} - -window.addEventListener("DOMContentLoaded", punchIn); diff --git a/app/client/src/RouteBuilder.ts b/app/client/src/RouteBuilder.ts index 4685dac00f..ca219dd6b3 100644 --- a/app/client/src/RouteBuilder.ts +++ b/app/client/src/RouteBuilder.ts @@ -2,6 +2,8 @@ import { ADMIN_SETTINGS_PATH, GEN_TEMPLATE_FORM_ROUTE, GEN_TEMPLATE_URL, + getViewerCustomPath, + getViewerPath, TEMPLATES_PATH, } from "constants/routes"; import { APP_MODE } from "entities/App"; @@ -26,8 +28,9 @@ export const fillPathname = ( page: Page, ) => { const replaceValue = page.customSlug - ? `/app/${page.customSlug}-${page.pageId}` - : `/app/${application.slug}/${page.slug}-${page.pageId}`; + ? getViewerCustomPath(page.customSlug, page.pageId) + : getViewerPath(application.slug, page.slug, page.pageId); + return pathname.replace( `/applications/${application.id}/pages/${page.pageId}`, replaceValue, diff --git a/app/client/src/ce/sagas/userSagas.tsx b/app/client/src/ce/sagas/userSagas.tsx index 419e2fe36d..ed27e49c58 100644 --- a/app/client/src/ce/sagas/userSagas.tsx +++ b/app/client/src/ce/sagas/userSagas.tsx @@ -55,7 +55,10 @@ import { getFirstTimeUserOnboardingApplicationId, getFirstTimeUserOnboardingIntroModalVisibility, } from "utils/storage"; -import { initializeAnalyticsAndTrackers } from "utils/AppsmithUtils"; +import { + initializeAnalyticsAndTrackers, + initializeSegmentWithoutTracking, +} from "utils/AppsmithUtils"; import { getAppsmithConfigs } from "ce/configs"; import { getSegmentState } from "selectors/analyticsSelectors"; import { @@ -64,6 +67,7 @@ import { } from "actions/analyticsActions"; import { SegmentState } from "reducers/uiReducers/analyticsReducer"; import FeatureFlags from "entities/FeatureFlags"; +import UsagePulse from "usagePulse"; export function* createUserSaga( action: ReduxActionWithPromise, @@ -134,19 +138,35 @@ export function* getCurrentUserSaga() { if (isValidResponse) { //@ts-expect-error: response is of type unknown const { enableTelemetry } = response.data; + if (enableTelemetry) { const promise = initializeAnalyticsAndTrackers(); + if (promise instanceof Promise) { const result: boolean = yield promise; + if (result) { yield put(segmentInitSuccess()); } else { yield put(segmentInitUncertain()); } } + } else if ( + //@ts-expect-error: response is of type unknown + response.data.isAnonymous && + //@ts-expect-error: response is of type unknown + response.data.username === ANONYMOUS_USERNAME + ) { + /* + * We're initializing the segment api regardless of the enableTelemetry flag + * So we can use segement Id to fingerprint anonymous user in usage pulse call + */ + yield initializeSegmentWithoutTracking(); } - yield put(initAppLevelSocketConnection()); - yield put(initPageLevelSocketConnection()); + + //To make sure that we're not tracking from previous session. + UsagePulse.stopTrackingActivity(); + if ( //@ts-expect-error: response is of type unknown !response.data.isAnonymous && @@ -155,15 +175,28 @@ export function* getCurrentUserSaga() { ) { //@ts-expect-error: response is of type unknown enableTelemetry && AnalyticsUtil.identifyUser(response.data); + } else { + UsagePulse.userAnonymousId = AnalyticsUtil.getAnonymousId(); + + if (!enableTelemetry) { + AnalyticsUtil.removeAnalytics(); + } } + + UsagePulse.startTrackingActivity(); + + yield put(initAppLevelSocketConnection()); + yield put(initPageLevelSocketConnection()); yield put({ type: ReduxActionTypes.FETCH_USER_DETAILS_SUCCESS, payload: response.data, }); + //@ts-expect-error: response is of type unknown if (response.data.emptyInstance) { history.replace(SETUP); } + PerformanceTracker.stopAsyncTracking( PerformanceTransactionName.USER_ME_API, ); @@ -421,6 +454,7 @@ export function* logoutSaga(action: ReduxAction<{ redirectURL: string }>) { const response: ApiResponse = yield call(UserApi.logoutUser); const isValidResponse: boolean = yield validateResponse(response); if (isValidResponse) { + UsagePulse.stopTrackingActivity(); AnalyticsUtil.reset(); const currentUser: User | undefined = yield select(getCurrentUser); yield put(logoutUserSuccess(!!currentUser?.emptyInstance)); diff --git a/app/client/src/constants/routes/appRoutes.ts b/app/client/src/constants/routes/appRoutes.ts index 8c48b19ffd..9a348e0859 100644 --- a/app/client/src/constants/routes/appRoutes.ts +++ b/app/client/src/constants/routes/appRoutes.ts @@ -3,12 +3,22 @@ // All solutions from closed issues on their repo have been tried. Ref: https://github.com/pillarjs/path-to-regexp/issues/193 const { match } = require("path-to-regexp"); +export const BUILDER_VIEWER_PATH_PREFIX = "/app/"; +export const BUILDER_PATH = `${BUILDER_VIEWER_PATH_PREFIX}:applicationSlug/:pageSlug(.*\-):pageId/edit`; +export const BUILDER_CUSTOM_PATH = `${BUILDER_VIEWER_PATH_PREFIX}:customSlug(.*\-):pageId/edit`; +export const VIEWER_PATH = `${BUILDER_VIEWER_PATH_PREFIX}:applicationSlug/:pageSlug(.*\-):pageId`; +export const VIEWER_CUSTOM_PATH = `${BUILDER_VIEWER_PATH_PREFIX}:customSlug(.*\-):pageId`; +export const getViewerPath = ( + applicationSlug: string, + pageSlug: string, + pageId: string, +) => `${BUILDER_VIEWER_PATH_PREFIX}${applicationSlug}/${pageSlug}-${pageId}`; +export const getViewerCustomPath = (customSlug: string, pageId: string) => + `${BUILDER_VIEWER_PATH_PREFIX}${customSlug}-${pageId}`; export const BUILDER_PATH_DEPRECATED = `/applications/:applicationId/pages/:pageId/edit`; -export const BUILDER_PATH = `/app/:applicationSlug/:pageSlug(.*\-):pageId/edit`; -export const VIEWER_PATH = `/app/:applicationSlug/:pageSlug(.*\-):pageId`; -export const BUILDER_CUSTOM_PATH = `/app/:customSlug(.*\-):pageId/edit`; -export const VIEWER_CUSTOM_PATH = `/app/:customSlug(.*\-):pageId`; export const VIEWER_PATH_DEPRECATED = `/applications/:applicationId/pages/:pageId`; +export const VIEWER_PATH_DEPRECATED_REGEX = /\/applications\/[^/]+\/pages\/[^/]+/; + export const VIEWER_FORK_PATH = `/fork`; export const INTEGRATION_EDITOR_PATH = `/datasources/:selectedTab`; export const API_EDITOR_BASE_PATH = `/api`; diff --git a/app/client/src/index.tsx b/app/client/src/index.tsx index 1b7838cd2a..3b555f9186 100755 --- a/app/client/src/index.tsx +++ b/app/client/src/index.tsx @@ -3,6 +3,7 @@ import "./wdyr"; import ReactDOM from "react-dom"; import { Provider } from "react-redux"; import "./index.css"; +import "./usagePulse"; import { ThemeProvider } from "constants/DefaultTheme"; import { appInitializer } from "utils/AppUtils"; import { Slide } from "react-toastify"; diff --git a/app/client/src/usagePulse/index.ts b/app/client/src/usagePulse/index.ts new file mode 100644 index 0000000000..1ec4f3f330 --- /dev/null +++ b/app/client/src/usagePulse/index.ts @@ -0,0 +1,118 @@ +import { + BUILDER_VIEWER_PATH_PREFIX, + VIEWER_PATH_DEPRECATED_REGEX, +} from "constants/routes"; +import { noop } from "lodash"; +import history from "utils/history"; + +const PULSE_API_ENDPOINT = "/api/v1/usage-pulse"; +const PULSE_INTERVAL = 3600; /* 1 hour in seconds */ +const USER_ACTIVITY_LISTENER_EVENTS = ["pointerdown", "keydown"]; +class UsagePulse { + static userAnonymousId: string | undefined; + static Timer: number; + static unlistenRouteChange: () => void; + + /* + * Function to check if the given URL is trakable or not. + * app builder and viewer urls are trackable + */ + static isTrackableUrl(url: string) { + return ( + url.includes(BUILDER_VIEWER_PATH_PREFIX) || + VIEWER_PATH_DEPRECATED_REGEX.test(url) + ); + } + + static sendPulse() { + const data: Record = { + viewMode: !window.location.href.endsWith("/edit"), + }; + + if (UsagePulse.userAnonymousId) { + data["anonymousUserId"] = UsagePulse.userAnonymousId; + } + + fetch(PULSE_API_ENDPOINT, { + method: "POST", + credentials: "same-origin", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(data), + keepalive: true, + }).catch(noop); + } + + static registerActivityListener() { + USER_ACTIVITY_LISTENER_EVENTS.forEach((event) => { + window.document.body.addEventListener( + event, + UsagePulse.startTrackingActivity, + ); + }); + } + + static deregisterActivityListener() { + USER_ACTIVITY_LISTENER_EVENTS.forEach((event) => { + window.document.body.removeEventListener( + event, + UsagePulse.startTrackingActivity, + ); + }); + } + + /* + * Function to register a history change event and trigger + * a callback and unlisten when the user goes to a trackable URL + */ + static watchForTrackableUrl(callback: () => void) { + UsagePulse.unlistenRouteChange = history.listen(() => { + if (UsagePulse.isTrackableUrl(window.location.href)) { + UsagePulse.unlistenRouteChange(); + setTimeout(callback, 0); + } + }); + + UsagePulse.deregisterActivityListener(); + } + + /* + * Function that suspends active tracking listeners + * and schedules when next listeners should be registered. + */ + static scheduleNextActivityListeners() { + UsagePulse.deregisterActivityListener(); + + UsagePulse.Timer = setTimeout( + UsagePulse.registerActivityListener, + PULSE_INTERVAL * 1000, + ); + } + + /* + * Point of entry for the user tracking + * triggers a pulse and schedules the pulse , if user is on a trackable url, otherwise + * registers listeners to wait for the user to go to a trackable url + */ + static startTrackingActivity() { + if (UsagePulse.isTrackableUrl(window.location.href)) { + UsagePulse.sendPulse(); + UsagePulse.scheduleNextActivityListeners(); + } else { + UsagePulse.watchForTrackableUrl(UsagePulse.startTrackingActivity); + } + } + + /* + * Function to cleanup states and listeners + */ + static stopTrackingActivity() { + UsagePulse.userAnonymousId = undefined; + clearTimeout(UsagePulse.Timer); + UsagePulse.unlistenRouteChange && UsagePulse.unlistenRouteChange(); + UsagePulse.deregisterActivityListener(); + } +} + +export default UsagePulse; diff --git a/app/client/src/usagePulse/usagePulse.test.ts b/app/client/src/usagePulse/usagePulse.test.ts new file mode 100644 index 0000000000..1f2b4aad74 --- /dev/null +++ b/app/client/src/usagePulse/usagePulse.test.ts @@ -0,0 +1,32 @@ +import UsagePulse from "usagePulse"; + +describe("Usage pulse", () => { + describe("isTrackableUrl", () => { + it("should return true when called with trackable URL", () => { + // All application URLS are trackable. + + [ + "https://dev.appsmith.com/app/test/mypage-123123/edit", + "https://dev.appsmith.com/app/test/mypage-123123", + "https://dev.appsmith.com/app/test-123123/edit", + "https://dev.appsmith.com/app/test-123123", + "https://dev.appsmith.com/applications/123123test/pages/123123test/edit", + "https://dev.appsmith.com/applications/123123test/pages/123123test", + ].forEach((url) => { + expect(UsagePulse.isTrackableUrl(url)).toBeTruthy(); + }); + }); + + it("should return false when called with untrackable URL", () => { + [ + "https://dev.appsmith.com/applications", + "https://dev.appsmith.com/login", + "https://dev.appsmith.com/signup", + "https://dev.appsmith.com/settings", + "https://dev.appsmith.com/generate-page", + ].forEach((url) => { + expect(UsagePulse.isTrackableUrl(url)).toBeFalsy(); + }); + }); + }); +}); diff --git a/app/client/src/utils/AnalyticsUtil.tsx b/app/client/src/utils/AnalyticsUtil.tsx index 2008468ec6..088eb7e921 100644 --- a/app/client/src/utils/AnalyticsUtil.tsx +++ b/app/client/src/utils/AnalyticsUtil.tsx @@ -317,11 +317,17 @@ class AnalyticsUtil { static cachedAnonymoustId: string; static cachedUserId: string; static user?: User = undefined; + static blockTrackEvent: boolean | undefined; static initializeSmartLook(id: string) { smartlookClient.init(id); } + static initializeSegmentWithoutTracking(key: string) { + AnalyticsUtil.blockTrackEvent = true; + return AnalyticsUtil.initializeSegment(key); + } + static initializeSegment(key: string) { const initPromise = new Promise((resolve) => { (function init(window: any) { @@ -388,6 +394,10 @@ class AnalyticsUtil { } static logEvent(eventName: EventName, eventData: any = {}) { + if (AnalyticsUtil.blockTrackEvent) { + return; + } + const windowDoc: any = window; let finalEventData = eventData; const userData = AnalyticsUtil.user; @@ -485,6 +495,8 @@ class AnalyticsUtil { username: userData.username, }); } + + AnalyticsUtil.blockTrackEvent = false; } static getAnonymousId() { @@ -506,6 +518,11 @@ class AnalyticsUtil { windowDoc.mixpanel && windowDoc.mixpanel.reset(); window.zipy && window.zipy.anonymize(); } + + static removeAnalytics() { + AnalyticsUtil.blockTrackEvent = false; + (window as any).analytics = undefined; + } } export default AnalyticsUtil; diff --git a/app/client/src/utils/AppsmithUtils.tsx b/app/client/src/utils/AppsmithUtils.tsx index abe7de71ca..bf8289c558 100644 --- a/app/client/src/utils/AppsmithUtils.tsx +++ b/app/client/src/utils/AppsmithUtils.tsx @@ -84,6 +84,24 @@ export const initializeAnalyticsAndTrackers = () => { } }; +export const initializeSegmentWithoutTracking = () => { + const appsmithConfigs = getAppsmithConfigs(); + + if (appsmithConfigs.segment.apiKey) { + // This value is only enabled for Appsmith's cloud hosted version. It is not set in self-hosted environments + return AnalyticsUtil.initializeSegmentWithoutTracking( + appsmithConfigs.segment.apiKey, + ); + } else if (appsmithConfigs.segment.ceKey) { + // This value is set in self-hosted environments. But if the analytics are disabled, it's never used. + return AnalyticsUtil.initializeSegmentWithoutTracking( + appsmithConfigs.segment.ceKey, + ); + } else { + return Promise.resolve(); + } +}; + export const mapToPropList = (map: Record): Property[] => { return _.map(map, (value, key) => { return { key: key, value: value };