Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: Server side observability (#19828)

Browse Source 
## Description

TL;DR: This PR introduces metrics logging using native Spring support
for Micrometer. It includes a docker-compose to set up all the required
parts of this observability stack in the local environment as well.

In order to make use of this stack, please navigate to
`utils/observability` and execute the following command:
```
docker-compose up -d
```

The set up comes bundled with a default Grafana dashboard that can be
accessed at localhost:3001. Please feel free to switch the mapping ports
around in the docker-compose file.

This dashboard currently shows all http requests (sampled at 0.1 by
default), and the server side implementation has introduced some minimal
tracing for the `/api/v1/action/execute` endpoint. This means that you
can use the trace id from http server requests for this endpoint to
delve deeper into the spans exposed in this flow.

In case you would like to send trace information to another service,
please make use of the `APPSMITH_TRACING_ENDPOINT` variable. To override
the default sampling rate in your local (to say, 1), you can set that as
the value for the variable `APPSMITH_SAMPLING_PROBABILITY`.

Fixes #19153

## Type of change
- Chore (housekeeping or task changes that don't impact user perception)

## How Has This Been Tested?
- Manual

### Test Plan
No testing required, only needs regression after merge.

## Checklist:
### Dev activity
- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [ ] PR is being merged under a feature flag

---------

Co-authored-by: Sumesh Pradhan <sumesh@appsmith.com>
This commit is contained in:
Nidhi 2023-02-07 15:56:18 +07:00 committed by GitHub
parent 8076b62c1c
commit 7e15d8b13d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 542 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -33,3 +33,7 @@ app/client/yalc.lock
.idea
.fleet/*
app/client/.fleet/*
# Observability related local storage
utils/observability/tempo-data/*

4
app/server/appsmith-interfaces/pom.xml
View File

@ -236,6 +236,10 @@
<artifactId>spring-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core-micrometer</artifactId>
</dependency>
</dependencies>
<build>

17
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/spans/ActionSpans.java vendored Normal file
View File

@ -0,0 +1,17 @@
package com.appsmith.external.constants.spans;
public final class ActionSpans {
// Action execution spans
public static final String ACTION_EXECUTION_REQUEST_PARSING = "request.parsing";
public static final String ACTION_EXECUTION_CACHED_ACTION = "get.action.cached";
public static final String ACTION_EXECUTION_CACHED_DATASOURCE = "get.datasource.cached";
public static final String ACTION_EXECUTION_CACHED_PLUGIN = "get.plugin.cached";
public static final String ACTION_EXECUTION_DATASOURCE_CONTEXT = "get.datasource.context";
public static final String ACTION_EXECUTION_DATASOURCE_CONTEXT_REMOTE = "get.datasource.context.remote";
public static final String ACTION_EXECUTION_EDITOR_CONFIG = "get.editorConfig.cached";
public static final String ACTION_EXECUTION_VALIDATE_AUTHENTICATION = "validate.authentication";
public static final String ACTION_EXECUTION_PLUGIN_EXECUTION = "total.plugin.execution";
public static final String ACTION_EXECUTION_SERVER_EXECUTION = "total.server.execution";
}

14
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java vendored
View File

@ -12,8 +12,10 @@ import com.appsmith.external.models.Param;
import com.appsmith.external.models.Property;
import com.appsmith.external.models.TriggerRequestDTO;
import com.appsmith.external.models.TriggerResultDTO;
import io.micrometer.observation.ObservationRegistry;
import org.pf4j.ExtensionPoint;
import org.springframework.util.CollectionUtils;
import reactor.core.observability.micrometer.Micrometer;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuple2;
@ -24,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static com.appsmith.external.constants.spans.ActionSpans.ACTION_EXECUTION_PLUGIN_EXECUTION;
import static com.appsmith.external.helpers.PluginUtils.getHintMessageForLocalhostUrl;
public interface PluginExecutor<C> extends ExtensionPoint, CrudTemplateService {
@ -171,6 +174,17 @@ public interface PluginExecutor<C> extends ExtensionPoint, CrudTemplateService {
return this.execute(connection, datasourceConfiguration, actionConfiguration);
}
default Mono<ActionExecutionResult> executeParameterizedWithMetrics(C connection,
ExecuteActionDTO executeActionDTO,
DatasourceConfiguration datasourceConfiguration,
ActionConfiguration actionConfiguration,
ObservationRegistry observationRegistry) {
return this.executeParameterized(connection, executeActionDTO, datasourceConfiguration, actionConfiguration)
.tag("plugin", this.getClass().getName())
.name(ACTION_EXECUTION_PLUGIN_EXECUTION)
.tap(Micrometer.observation(observationRegistry));
}
/**
* This function is responsible for preparing the action and datasource configurations to be ready for execution.
*

11
app/server/appsmith-server/pom.xml
View File

@ -203,10 +203,15 @@
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>context-propagation</artifactId>
<version>1.0.0</version>
<groupId>io.zipkin.reporter2</groupId>
<artifactId>zipkin-reporter-brave</artifactId>
</dependency>
<!-- Commented oout Loki dependency for now, since we haven't fixed associating logs to traces-->
<!-- <dependency>-->
<!-- <groupId>com.github.loki4j</groupId>-->
<!-- <artifactId>loki-logback-appender</artifactId>-->
<!-- <version>1.3.2</version>-->
<!-- </dependency>-->
<!-- Actual Junit5 implementation. Will transitively include junit-jupiter-api -->
<dependency>
<groupId>org.junit.jupiter</groupId>

2
app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/InstanceConfig.java
View File

@ -101,7 +101,7 @@ public class InstanceConfig implements ApplicationListener<ApplicationReadyEvent
// Keep adding version numbers that brought in breaking instance schema migrations here
switch (currentInstanceSchemaVersion) {
// Example, we expect that in v1.8.14, all instances will have been migrated to instanceSchemaVer 2
// Example, we expect that in v1.9.2, all instances will have been migrated to instanceSchemaVer 2
case 1:
versions.add("v1.9.2");
docs.add("https://docs.appsmith.com/help-and-support/troubleshooting-guide/deployment-errors#server-shuts-down-with-schema-mismatch-error");

2
app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/SecurityConfig.java
View File

@ -146,7 +146,7 @@ public class SecurityConfig {
ServerWebExchangeMatchers.pathMatchers(HttpMethod.GET, CUSTOM_JS_LIB_URL + "/*/view")
)
.permitAll()
.pathMatchers("/public/**", "/oauth2/**").permitAll()
.pathMatchers("/public/**", "/oauth2/**", "/actuator/**").permitAll()
.anyExchange()
.authenticated()
.and()

6
app/server/appsmith-server/src/main/java/com/appsmith/server/services/NewActionServiceImpl.java
View File

@ -10,6 +10,7 @@ import com.appsmith.server.solutions.ActionPermission;
import com.appsmith.server.solutions.ApplicationPermission;
import com.appsmith.server.solutions.DatasourcePermission;
import com.appsmith.server.solutions.PagePermission;
import io.micrometer.observation.ObservationRegistry;
import jakarta.validation.Validator;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.mongodb.core.ReactiveMongoTemplate;
@ -44,13 +45,14 @@ public class NewActionServiceImpl extends NewActionServiceCEImpl implements NewA
DatasourcePermission datasourcePermission,
ApplicationPermission applicationPermission,
PagePermission pagePermission,
ActionPermission actionPermission) {
ActionPermission actionPermission,
ObservationRegistry observationRegistry) {
super(scheduler, validator, mongoConverter, reactiveMongoTemplate, repository, analyticsService,
datasourceService, pluginService, datasourceContextService, pluginExecutorHelper, marketplaceService,
policyGenerator, newPageService, applicationService, sessionUserService, policyUtils,
authenticationValidator, configService, responseUtils, permissionGroupService, datasourcePermission,
applicationPermission, pagePermission, actionPermission);
applicationPermission, pagePermission, actionPermission, observationRegistry);
}
}

47
app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/NewActionServiceCEImpl.java
View File

@ -68,6 +68,7 @@ import com.appsmith.server.solutions.PagePermission;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.micrometer.observation.ObservationRegistry;
import jakarta.validation.Validator;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.ArrayUtils;
@ -83,6 +84,7 @@ import org.springframework.util.LinkedCaseInsensitiveMap;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StringUtils;
import reactor.core.observability.micrometer.Micrometer;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
@ -111,6 +113,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import static com.appsmith.external.constants.CommonFieldName.REDACTED_DATA;
import static com.appsmith.external.constants.spans.ActionSpans.*;
import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNewFieldValuesIntoOldObject;
import static com.appsmith.external.helpers.DataTypeStringUtils.getDisplayDataTypes;
import static com.appsmith.external.helpers.PluginUtils.setValueSafelyInFormData;
@ -154,6 +157,8 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
private final PagePermission pagePermission;
private final ActionPermission actionPermission;
private final ObservationRegistry observationRegistry;
public NewActionServiceCEImpl(Scheduler scheduler,
Validator validator,
MongoConverter mongoConverter,
@ -177,7 +182,8 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
DatasourcePermission datasourcePermission,
ApplicationPermission applicationPermission,
PagePermission pagePermission,
ActionPermission actionPermission) {
ActionPermission actionPermission,
ObservationRegistry observationRegistry) {
super(scheduler, validator, mongoConverter, reactiveMongoTemplate, repository, analyticsService);
this.repository = repository;
@ -193,6 +199,7 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
this.policyUtils = policyUtils;
this.authenticationValidator = authenticationValidator;
this.permissionGroupService = permissionGroupService;
this.observationRegistry = observationRegistry;
this.objectMapper = new ObjectMapper();
this.responseUtils = responseUtils;
this.configService = configService;
@ -664,6 +671,8 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
return repository.findById(actionId, actionPermission.getExecutePermission())
.switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.ACTION, actionId)))
.name(ACTION_EXECUTION_CACHED_ACTION)
.tap(Micrometer.observation(observationRegistry))
.cache();
}
@ -703,6 +712,8 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
// The external datasource have already been validated. No need to validate again.
return Mono.just(datasource);
})
.name(ACTION_EXECUTION_CACHED_DATASOURCE)
.tap(Micrometer.observation(observationRegistry))
.cache();
}
@ -727,6 +738,8 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
return pluginService.findById(datasource.getPluginId());
})
.switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.PLUGIN)))
.name(ACTION_EXECUTION_CACHED_PLUGIN)
.tap(Micrometer.observation(observationRegistry))
.cache();
}
@ -745,7 +758,9 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
}
return pluginService.getEditorConfigLabelMap(datasource.getPluginId());
});
})
.name(ACTION_EXECUTION_EDITOR_CONFIG)
.tap(Micrometer.observation(observationRegistry));
}
/**
@ -795,10 +810,11 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
Instant requestedAt = Instant.now();
return ((Mono<ActionExecutionResult>)
pluginExecutor.executeParameterized(resourceContext.getConnection(),
pluginExecutor.executeParameterizedWithMetrics(resourceContext.getConnection(),
executeActionDTO,
validatedDatasource.getDatasourceConfiguration(),
actionDTO.getActionConfiguration()))
actionDTO.getActionConfiguration(),
observationRegistry))
.map(actionExecutionResult -> {
ActionExecutionRequest actionExecutionRequest = actionExecutionResult.getRequest();
if (actionExecutionRequest == null) {
@ -853,7 +869,10 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
protected Mono<Datasource> getValidatedDatasourceForActionExecution(Datasource datasource, String environmentId) {
// the environmentName argument is not consumed over here
// See EE override for usage of variable
return authenticationValidator.validateAuthentication(datasource, environmentId).cache();
return authenticationValidator.validateAuthentication(datasource, environmentId)
.name(ACTION_EXECUTION_VALIDATE_AUTHENTICATION)
.tap(Micrometer.observation(observationRegistry))
.cache();
}
/**
@ -869,9 +888,15 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
DatasourceContextIdentifier datasourceContextIdentifier,
Map<String, BaseDomain> environmentMap) {
if (plugin.isRemotePlugin()) {
return datasourceContextService.getRemoteDatasourceContext(plugin, validatedDatasource);
return datasourceContextService.getRemoteDatasourceContext(plugin, validatedDatasource)
.tag("plugin", plugin.getPackageName())
.name(ACTION_EXECUTION_DATASOURCE_CONTEXT_REMOTE)
.tap(Micrometer.observation(observationRegistry));
}
return datasourceContextService.getDatasourceContext(validatedDatasource, datasourceContextIdentifier, environmentMap);
return datasourceContextService.getDatasourceContext(validatedDatasource, datasourceContextIdentifier, environmentMap)
.tag("plugin", plugin.getPackageName())
.name(ACTION_EXECUTION_DATASOURCE_CONTEXT)
.tap(Micrometer.observation(observationRegistry));
}
/**
@ -1188,7 +1213,9 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
}
dto.setParams(params);
return Mono.just(dto);
});
})
.name(ACTION_EXECUTION_REQUEST_PARSING)
.tap(Micrometer.observation(observationRegistry));
}
/**
@ -1209,7 +1236,9 @@ public class NewActionServiceCEImpl extends BaseService<NewActionRepository, New
executeActionDTO.setActionId(branchedAction.getId());
return executeActionDTO;
}))
.flatMap(executeActionDTO -> this.executeAction(executeActionDTO, environmentName));
.flatMap(executeActionDTO -> this.executeAction(executeActionDTO, environmentName))
.name(ACTION_EXECUTION_SERVER_EXECUTION)
.tap(Micrometer.observation(observationRegistry));
}

3
app/server/appsmith-server/src/main/java/com/appsmith/server/solutions/ce/PageLoadActionsUtilCEImpl.java
View File

@ -788,7 +788,8 @@ public class PageLoadActionsUtilCEImpl implements PageLoadActionsUtilCE {
Set<String> bindingPaths = actionBindingMap.keySet();
return Flux.fromIterable(bindingPaths).flatMap(bindingPath -> {
return Flux.fromIterable(bindingPaths)
.flatMap(bindingPath -> {
EntityDependencyNode actionDependencyNode = new EntityDependencyNode(entityDependencyNode.getEntityReferenceType(), entityDependencyNode.getValidEntityName(), bindingPath, null, false, action);
return getPossibleEntityReferences(actionNameToActionMapMono, actionBindingMap.get(bindingPath), evalVersion, bindingsInDsl)
.flatMapMany(Flux::fromIterable)

13
app/server/appsmith-server/src/main/resources/application.properties
View File

@ -50,7 +50,7 @@ segment.ce.key = ${APPSMITH_SEGMENT_CE_KEY:}
# Sentry
sentry.dsn=${APPSMITH_SENTRY_DSN:}
sentry.send-default-pii=true
sentry.debug=off
sentry.debug=false
sentry.environment=${APPSMITH_SENTRY_ENVIRONMENT:}
# Redis Properties
@ -87,13 +87,12 @@ encrypt.password=${APPSMITH_ENCRYPTION_PASSWORD:}
encrypt.salt=${APPSMITH_ENCRYPTION_SALT:}
# The following configurations are to help support prometheus scraping for monitoring
management.endpoints.web.exposure.include=prometheus
management.metrics.web.server.request.autotime.enabled=true
management.endpoints.web.exposure.include=prometheus,metrics
management.tracing.enabled=${APPSMITH_TRACING_ENABLED:false}
management.zipkin.tracing.endpoint=${APPSMITH_TRACING_ENDPOINT:http://localhost:9411}/api/v2/spans
management.tracing.sampling.probability=${APPSMITH_SAMPLING_PROBABILITY:0.1}
management.prometheus.metrics.export.descriptions=true
management.metrics.web.server.request.ignore-trailing-slash=true
management.metrics.web.server.request.autotime.percentiles=0.5, 0.9, 0.95, 0.99
management.metrics.web.server.request.autotime.percentiles-histogram=true
management.metrics.distribution.sla.[http.server.requests]=1s
management.metrics.distribution.percentiles-histogram.http.server.requests=true
# Support disabling signup with an environment variable
signup.disabled = ${APPSMITH_SIGNUP_DISABLED:false}

7
app/server/appsmith-server/src/test/java/com/appsmith/server/services/ApplicationTemplateServiceTest.java
View File

@ -144,15 +144,14 @@ public class ApplicationTemplateServiceTest {
// make sure we've received the response returned by the mockCloudServices
StepVerifier.create(applicationTemplateService.getRecentlyUsedTemplates())
.assertNext(applicationTemplates -> assertThat(applicationTemplates.size()).isEqualTo(1))
.assertNext(applicationTemplates -> assertThat(applicationTemplates).hasSize(1))
.verifyComplete();
// verify that mockCloudServices was called with the query param id i.e. id=id-one&id=id-two
RecordedRequest recordedRequest = mockCloudServices.takeRequest();
assert recordedRequest.getRequestUrl() != null;
List<String> queryParameterValues = recordedRequest.getRequestUrl().queryParameterValues("id");
assertThat(queryParameterValues).contains("id-one");
assertThat(queryParameterValues).contains("id-two");
assertThat(queryParameterValues.size()).isEqualTo(2);
assertThat(queryParameterValues).containsExactly("id-one", "id-two");
}
@Test

12
app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/ActionServiceCE_Test.java
View File

@ -839,7 +839,7 @@ public class ActionServiceCE_Test {
AppsmithPluginException pluginException = new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR);
Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(pluginExecutor));
Mockito.when(pluginExecutor.executeParameterized(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(Mono.error(pluginException));
Mockito.when(pluginExecutor.executeParameterizedWithMetrics(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(Mono.error(pluginException));
Mockito.when(pluginExecutor.datasourceCreate(Mockito.any())).thenReturn(Mono.empty());
Mono<ActionExecutionResult> executionResultMono = newActionService.executeAction(executeActionDTO, null);
@ -889,7 +889,7 @@ public class ActionServiceCE_Test {
AppsmithPluginException pluginException = new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR);
Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(pluginExecutor));
Mockito.when(pluginExecutor.executeParameterized(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(Mono.error(pluginException));
Mockito.when(pluginExecutor.executeParameterizedWithMetrics(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(Mono.error(pluginException));
Mockito.when(pluginExecutor.datasourceCreate(Mockito.any())).thenReturn(Mono.empty());
Mono<ActionExecutionResult> executionResultMono = newActionService.executeAction(executeActionDTO, null);
@ -932,7 +932,7 @@ public class ActionServiceCE_Test {
executeActionDTO.setViewMode(false);
Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(pluginExecutor));
Mockito.when(pluginExecutor.executeParameterized(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
Mockito.when(pluginExecutor.executeParameterizedWithMetrics(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
.thenReturn(Mono.error(new StaleConnectionException())).thenReturn(Mono.error(new StaleConnectionException()));
Mockito.when(pluginExecutor.datasourceCreate(Mockito.any())).thenReturn(Mono.empty());
@ -976,7 +976,7 @@ public class ActionServiceCE_Test {
executeActionDTO.setViewMode(false);
Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(pluginExecutor));
Mockito.when(pluginExecutor.executeParameterized(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
Mockito.when(pluginExecutor.executeParameterizedWithMetrics(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
.thenAnswer(x -> Mono.delay(Duration.ofMillis(1000)).ofType(ActionExecutionResult.class));
Mockito.when(pluginExecutor.datasourceCreate(Mockito.any())).thenReturn(Mono.empty());
@ -1061,7 +1061,7 @@ public class ActionServiceCE_Test {
mockResult.setBody("response-body");
Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(pluginExecutor));
Mockito.when(pluginExecutor.executeParameterized(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
Mockito.when(pluginExecutor.executeParameterizedWithMetrics(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
.thenThrow(new StaleConnectionException())
.thenReturn(Mono.just(mockResult));
Mockito.when(pluginExecutor.datasourceCreate(Mockito.any())).thenReturn(Mono.empty());
@ -1114,7 +1114,7 @@ public class ActionServiceCE_Test {
private Mono<ActionExecutionResult> executeAction(ExecuteActionDTO executeActionDTO, ActionConfiguration actionConfiguration, ActionExecutionResult mockResult) {
Mockito.when(pluginExecutorHelper.getPluginExecutor(Mockito.any())).thenReturn(Mono.just(pluginExecutor));
Mockito.when(pluginExecutor.executeParameterized(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(Mono.just(mockResult));
Mockito.when(pluginExecutor.executeParameterizedWithMetrics(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(Mono.just(mockResult));
Mockito.when(pluginExecutor.datasourceCreate(Mockito.any())).thenReturn(Mono.empty());
Mono<ActionExecutionResult> actionExecutionResultMono = newActionService.executeAction(executeActionDTO, null);

10
app/server/appsmith-server/src/test/java/com/appsmith/server/services/ce/NewActionServiceCEImplTest.java
View File

@ -165,7 +165,11 @@ public class NewActionServiceCEImplTest {
datasourcePermission,
applicationPermission,
pagePermission,
actionPermission);
actionPermission,
observationRegistry);
ObservationRegistry.ObservationConfig mockObservationConfig = Mockito.mock(ObservationRegistry.ObservationConfig.class);
Mockito.when(observationRegistry.observationConfig()).thenReturn(mockObservationConfig);
}
@BeforeEach
@ -349,8 +353,8 @@ public class NewActionServiceCEImplTest {
StepVerifier
.create(actionExecutionResultMono)
.assertNext(response -> {
assertTrue(response.getIsExecutionSuccess());
assertTrue(response instanceof ActionExecutionResult);
assertTrue(response.getIsExecutionSuccess());
assertEquals(mockResult.getBody().toString(), response.getBody().toString());
})
.verifyComplete();
@ -399,8 +403,8 @@ public class NewActionServiceCEImplTest {
StepVerifier
.create(actionExecutionResultMono)
.assertNext(response -> {
assertTrue(response.getIsExecutionSuccess());
assertTrue(response instanceof ActionExecutionResult);
assertTrue(response.getIsExecutionSuccess());
assertEquals(mockResult.getBody().toString(), response.getBody().toString());
})
.verifyComplete();

54
utils/observability/docker-compose.yml Normal file
View File

@ -0,0 +1,54 @@
networks:
default:
name: operations-dc
services:
tempo:
image: grafana/tempo
extra_hosts: ['host.docker.internal:host-gateway']
command: [ "-config.file=/etc/tempo.yaml" ]
volumes:
- ./docker/tempo/tempo-local.yaml:/etc/tempo.yaml:ro
- ./tempo-data:/tmp/tempo
ports:
- "14268" # jaeger ingest
- "9411:9411" # zipkin
- "3200:3200"
# loki:
# image: grafana/loki
# extra_hosts: ['host.docker.internal:host-gateway']
# command: [ "-config.file=/etc/loki/local-config.yaml" ]
# ports:
# - "3100:3100" # loki needs to be exposed so it receives logs
# environment:
# - JAEGER_AGENT_HOST=tempo
# - JAEGER_ENDPOINT=http://tempo:14268/api/traces # send traces to Tempo
# - JAEGER_SAMPLER_TYPE=const
# - JAEGER_SAMPLER_PARAM=1
prometheus:
image: prom/prometheus
extra_hosts: ['host.docker.internal:host-gateway']
command:
- --enable-feature=exemplar-storage
- --config.file=/etc/prometheus/prometheus.yml
volumes:
- ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
ports:
- "9090:9090"
grafana:
image: grafana/grafana
extra_hosts: ['host.docker.internal:host-gateway']
volumes:
- ./docker/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
- ./docker/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
environment:
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_AUTH_DISABLE_LOGIN_FORM=true
ports:
- "3001:3000"
# Prometheus: http://localhost:9090/
# Grafana: http://localhost:3000/

10
utils/observability/docker/grafana/provisioning/dashboards/dashboard.yml Normal file
View File

@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: dashboards
type: file
disableDeletion: true
editable: true
options:
path: /etc/grafana/provisioning/dashboards
foldersFromFilesStructure: true

294
utils/observability/docker/grafana/provisioning/dashboards/logs_traces_metrics.json Normal file
View File

@ -0,0 +1,294 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 6,
"iteration": 1654517000502,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"description": "",
"gridPos": {
"h": 10,
"w": 23,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": true,
"showCommonLabels": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Ascending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "builder",
"expr": "{traceID=\"$traceID\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs with trace ID $traceID",
"type": "logs"
},
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"description": "",
"gridPos": {
"h": 15,
"w": 23,
"x": 0,
"y": 10
},
"id": 6,
"targets": [
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"query": "$traceID",
"queryType": "traceId",
"refId": "A"
}
],
"title": "Trace View for trace with id $traceID",
"type": "traces"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 23,
"x": 0,
"y": 25
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": true,
"expr": "histogram_quantile(1.00, sum(rate(http_server_requests_seconds_bucket{uri=~\".*\"}[$__rate_interval])) by (le))",
"legendFormat": "max",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": true,
"expr": "histogram_quantile(0.99, sum(rate(http_server_requests_seconds_bucket{uri=~\".*\"}[$__rate_interval])) by (le))",
"hide": false,
"legendFormat": "tp99",
"range": true,
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": true,
"expr": "histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket{uri=~\".*\"}[$__rate_interval])) by (le))",
"hide": false,
"legendFormat": "tp95",
"range": true,
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": true,
"expr": "histogram_quantile(1.00, sum(rate(server_job_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "max",
"range": true,
"refId": "D"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": true,
"expr": "histogram_quantile(0.99, sum(rate(server_job_seconds_bucket[$__rate_interval])) by (le))",
"hide": false,
"legendFormat": "tp99",
"range": true,
"refId": "E"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": true,
"expr": "histogram_quantile(0.95, sum(rate(server_job_seconds_bucket[$__rate_interval])) by (le))",
"hide": false,
"legendFormat": "tp95",
"range": true,
"refId": "F"
}
],
"title": "latency for All",
"type": "timeseries"
}
],
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "0003776c79e02b6c",
"value": "0003776c79e02b6c"
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"definition": "label_values(traceID)",
"hide": 0,
"includeAll": false,
"label": "Trace ID",
"multi": false,
"name": "traceID",
"options": [],
"query": "label_values(traceID)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}
]
},
"time": {
"from": "now-15m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Logs, Traces, Metrics",
"uid": "szVLMe97z",
"version": 7,
"weekStart": ""
}

45
utils/observability/docker/grafana/provisioning/datasources/datasource.yml Normal file
View File

@ -0,0 +1,45 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://host.docker.internal:9090
editable: false
jsonData:
httpMethod: POST
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: 'tempo'
- name: Tempo
type: tempo
access: proxy
orgId: 1
url: http://tempo:3200
basicAuth: false
isDefault: true
version: 1
editable: false
apiVersion: 1
uid: tempo
jsonData:
httpMethod: GET
tracesToLogs:
datasourceUid: 'loki'
- name: Loki
type: loki
uid: loki
access: proxy
orgId: 1
url: http://loki:3100
basicAuth: false
isDefault: false
version: 1
editable: false
apiVersion: 1
jsonData:
derivedFields:
- datasourceUid: 'tempo'
matcherRegex: \[.+,(.+?),
name: TraceID
url: $${__value.raw}

12
utils/observability/docker/prometheus/prometheus.yml Normal file
View File

@ -0,0 +1,12 @@
global:
scrape_interval: 2s
evaluation_interval: 2s
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['host.docker.internal:9090']
- job_name: 'apps'
metrics_path: '/actuator/prometheus'
static_configs:
- targets: ['host.docker.internal:8080','host.docker.internal:8989']

12
utils/observability/docker/tempo/tempo-local.yaml Normal file
View File

@ -0,0 +1,12 @@
server:
http_listen_port: 3200
distributor:
receivers:
zipkin:
storage:
trace:
backend: local
local:
path: /tmp/tempo/blocks