Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore: v1.8.1 promotion (#16876)

Browse Source
This commit is contained in:
Arpit Mohan 2022-09-19 14:35:41 +02:00 committed by GitHub
commit f211d7d3b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 127 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
.github/workflows/test-build-docker-image.yml vendored
View File

@ -383,9 +383,8 @@ jobs:
run: |
ls -al dist/node_modules/@shared/ast
tar -cvf rts-dist.tar dist
# Upload the build artifact so that it can be used by the test & deploy job in the workflow
- name: Upload server build bundle
# Upload the build artifacts and dependencies so that it can be used by the test & deploy job in other workflows
- name: Upload rts build bundle
uses: actions/upload-artifact@v2
with:
name: rts-dist
@ -727,14 +726,14 @@ jobs:
- name: Download the rts build artifact
uses: actions/download-artifact@v2
with:
name: rts-build
name: rts-dist
path: app/rts/dist
- name: Download the rts build artifact
uses: actions/download-artifact@v2
with:
name: rts-build-deps
path: app/rts/node_modules/
- name: Untar the rts folder
run: |
tar -xvf app/rts/dist/rts-dist.tar -C app/rts/
echo "Cleaning up the tar files"
rm app/rts/dist/rts-dist.tar
# We don't use Depot Docker builds because it's faster for local Docker images to be built locally.
# It's slower and more expensive to build these Docker images on Depot and download it back to the CI node.
@ -1073,7 +1072,7 @@ jobs:
with:
path: |
~/run_result
key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}-${{ matrix.job }}
key: ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }}
restore-keys: |
${{ github.run_id }}-${{ github.job }}-${{ matrix.job }}
@ -1089,7 +1088,7 @@ jobs:
with:
path: |
~/combined_failed_spec
key: ${{ github.run_id }}-"ui-test-result"-${{ steps.timestamp.outputs.timestamp }}
key: ${{ github.run_id }}-"ui-test-result"
restore-keys: |
${{ github.run_id }}-${{ github.job }}
@ -1374,7 +1373,7 @@ jobs:
with:
path: |
~/run_result
key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}-${{ matrix.job }}
key: ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }}
restore-keys: |
${{ github.run_id }}-${{ github.job }}-${{ matrix.job }}
@ -1385,7 +1384,7 @@ jobs:
with:
path: |
~/failed_spec
key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}-${{ matrix.job }}
key: ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }}
restore-keys: |
${{ github.run_id }}-${{ github.job }}-${{ matrix.job }}
@ -1401,7 +1400,7 @@ jobs:
with:
path: |
app/client/cypress/snapshots/
key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}-${{ matrix.job }}
key: ${{ github.run_id }}-${{ github.job }}-${{ matrix.job }}
restore-keys: |
${{ github.run_id }}-${{ github.job }}-${{ matrix.job }}
@ -1457,7 +1456,7 @@ jobs:
with:
path: |
~/combined_failed_spec
key: ${{ github.run_id }}-"ui-test-result"-${{ steps.timestamp.outputs.timestamp }}
key: ${{ github.run_id }}-"ui-test-result"
restore-keys: |
${{ github.run_id }}-${{ github.job }}

11
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/restApiUtils/helpers/TriggerUtils.java vendored
View File

@ -35,10 +35,8 @@ import reactor.netty.resources.ConnectionProvider;
import javax.crypto.SecretKey;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
@ -46,7 +44,6 @@ import java.util.Date;
import java.util.List;
import java.util.Set;
import static com.appsmith.external.helpers.restApiUtils.helpers.URIUtils.DISALLOWED_HOSTS;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
@NoArgsConstructor
@ -204,14 +201,10 @@ public class TriggerUtils {
* It redirects to partial URI : /api/character/
* In this scenario we should convert the partial URI to complete URI
*/
URI redirectUri;
final URI redirectUri;
try {
redirectUri = new URI(redirectUrl);
if (DISALLOWED_HOSTS.contains(redirectUri.getHost())
|| DISALLOWED_HOSTS.contains(InetAddress.getByName(redirectUri.getHost()).getHostAddress())) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, "Host not allowed."));
}
} catch (URISyntaxException | UnknownHostException e) {
} catch (URISyntaxException e) {
return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, e));
}

16
app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/restApiUtils/helpers/URIUtils.java vendored
View File

@ -3,30 +3,21 @@ package com.appsmith.external.helpers.restApiUtils.helpers;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.DatasourceConfiguration;
import com.appsmith.external.models.Property;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.web.util.UriComponentsBuilder;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import static org.apache.commons.collections.CollectionUtils.isEmpty;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
@NoArgsConstructor
public class URIUtils {
public static final Set<String> DISALLOWED_HOSTS = Set.of(
"169.254.169.254",
"metadata.google.internal"
);
public URI createUriWithQueryParams(ActionConfiguration actionConfiguration,
DatasourceConfiguration datasourceConfiguration, String url,
@ -53,7 +44,7 @@ public class URIUtils {
for (Property queryParam : allQueryParams) {
String key = queryParam.getKey();
if (isNotEmpty(key)) {
if (encodeParamsToggle == true) {
if (encodeParamsToggle) {
uriBuilder.queryParam(
URLEncoder.encode(key, StandardCharsets.UTF_8),
URLEncoder.encode((String) queryParam.getValue(), StandardCharsets.UTF_8)
@ -78,9 +69,4 @@ public class URIUtils {
return "http://" + url;
}
public boolean isHostDisallowed(URI uri) throws UnknownHostException {
String host = uri.getHost();
return StringUtils.isEmpty(host) || DISALLOWED_HOSTS.contains(host)
|| DISALLOWED_HOSTS.contains(InetAddress.getByName(host).getHostAddress());
}
}

97
app/server/appsmith-interfaces/src/main/java/com/appsmith/util/WebClientUtils.java
View File

@ -1,11 +1,31 @@
package com.appsmith.util;
import io.netty.resolver.AddressResolver;
import io.netty.resolver.AddressResolverGroup;
import io.netty.resolver.InetNameResolver;
import io.netty.resolver.InetSocketAddressResolver;
import io.netty.util.concurrent.EventExecutor;
import io.netty.util.concurrent.Promise;
import io.netty.util.internal.SocketUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
public class WebClientUtils {
private static final Set<String> DISALLOWED_HOSTS = Set.of(
"169.254.169.254",
"metadata.google.internal"
);
private WebClientUtils() {
}
@ -31,15 +51,86 @@ public class WebClientUtils {
public static WebClient.Builder builder(HttpClient httpClient) {
return WebClient.builder()
.clientConnector(new ReactorClientHttpConnector(applyProxyIfConfigured(httpClient)));
.clientConnector(new ReactorClientHttpConnector(makeSafeHttpClient(httpClient)));
}
private static HttpClient applyProxyIfConfigured(HttpClient httpClient) {
private static HttpClient makeSafeHttpClient(HttpClient httpClient) {
if (shouldUseSystemProxy()) {
httpClient = httpClient.proxyWithSystemProperties();
}
return httpClient;
return httpClient.resolver(ResolverGroup.INSTANCE);
}
private static class ResolverGroup extends AddressResolverGroup<InetSocketAddress> {
public static final ResolverGroup INSTANCE = new ResolverGroup();
@Override
protected AddressResolver<InetSocketAddress> newResolver(EventExecutor executor) {
return new InetSocketAddressResolver(executor, new NameResolver(executor));
}
}
@Slf4j
private static class NameResolver extends InetNameResolver {
public NameResolver(EventExecutor executor) {
super(executor);
}
private static boolean isDisallowedAndFail(String host, Promise<?> promise) {
if (DISALLOWED_HOSTS.contains(host)) {
log.warn("Host {} is disallowed. Failing the request.", host);
promise.setFailure(new UnknownHostException("Host not allowed."));
return true;
}
return false;
}
@Override
protected void doResolve(String inetHost, Promise<InetAddress> promise) {
if (isDisallowedAndFail(inetHost, promise)) {
return;
}
final InetAddress address;
try {
address = SocketUtils.addressByName(inetHost);
} catch (UnknownHostException e) {
promise.setFailure(e);
return;
}
if (isDisallowedAndFail(address.getHostAddress(), promise)) {
return;
}
promise.setSuccess(address);
}
@Override
protected void doResolveAll(String inetHost, Promise<List<InetAddress>> promise) {
if (isDisallowedAndFail(inetHost, promise)) {
return;
}
final List<InetAddress> addresses;
try {
addresses = Arrays.asList(SocketUtils.allAddressesByName(inetHost));
} catch (UnknownHostException e) {
promise.setFailure(e);
return;
}
// Even if _one_ of the addresses is disallowed, we fail the request.
for (InetAddress address : addresses) {
if (isDisallowedAndFail(address.getHostAddress(), promise)) {
return;
}
}
promise.setSuccess(addresses);
}
}
}

19
app/server/appsmith-plugins/graphqlPlugin/src/main/java/com/external/plugins/GraphQLPlugin.java vendored
View File

@ -27,7 +27,6 @@ import reactor.core.publisher.Mono;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@ -37,12 +36,12 @@ import java.util.Set;
import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromPropertyList;
import static com.appsmith.external.helpers.PluginUtils.setValueSafelyInPropertyList;
import static com.external.utils.GraphQLBodyUtils.PAGINATION_DATA_INDEX;
import static com.external.utils.GraphQLDataTypeUtils.smartlyReplaceGraphQLQueryBodyPlaceholderWithValue;
import static com.external.utils.GraphQLPaginationUtils.updateVariablesWithPaginationValues;
import static com.external.utils.GraphQLBodyUtils.QUERY_VARIABLES_INDEX;
import static com.external.utils.GraphQLBodyUtils.convertToGraphQLPOSTBodyFormat;
import static com.external.utils.GraphQLBodyUtils.getGraphQLQueryParamsForBodyAndVariables;
import static com.external.utils.GraphQLBodyUtils.validateBodyAndVariablesSyntax;
import static com.external.utils.GraphQLDataTypeUtils.smartlyReplaceGraphQLQueryBodyPlaceholderWithValue;
import static com.external.utils.GraphQLPaginationUtils.updateVariablesWithPaginationValues;
import static java.lang.Boolean.TRUE;
import static org.apache.commons.lang3.StringUtils.isBlank;
@ -186,18 +185,6 @@ public class GraphQLPlugin extends BasePlugin {
ActionExecutionRequest actionExecutionRequest =
RequestCaptureFilter.populateRequestFields(actionConfiguration, uri, insertedParams, objectMapper);
try {
if (uriUtils.isHostDisallowed(uri)) {
errorResult.setBody(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR.getMessage("Host not allowed."));
errorResult.setRequest(actionExecutionRequest);
return Mono.just(errorResult);
}
} catch (UnknownHostException e) {
errorResult.setBody(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR.getMessage("Unknown host."));
errorResult.setRequest(actionExecutionRequest);
return Mono.just(errorResult);
}
WebClient.Builder webClientBuilder = triggerUtils.getWebClientBuilder(actionConfiguration,
datasourceConfiguration);
@ -282,7 +269,7 @@ public class GraphQLPlugin extends BasePlugin {
EXCHANGE_STRATEGIES, requestCaptureFilter);
/* Triggering the actual REST API call */
Set<String> hintMessages = new HashSet<String>();
Set<String> hintMessages = new HashSet<>();
return triggerUtils.triggerApiCall(client, httpMethod, uri, requestBodyObj, actionExecutionRequest,
objectMapper,
hintMessages, errorResult, requestCaptureFilter);

20
app/server/appsmith-plugins/restApiPlugin/src/main/java/com/external/plugins/RestApiPlugin.java vendored
View File

@ -5,6 +5,8 @@ import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError;
import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException;
import com.appsmith.external.helpers.DataTypeStringUtils;
import com.appsmith.external.helpers.MustacheHelper;
import com.appsmith.external.helpers.restApiUtils.connections.APIConnection;
import com.appsmith.external.helpers.restApiUtils.helpers.RequestCaptureFilter;
import com.appsmith.external.models.ActionConfiguration;
import com.appsmith.external.models.ActionExecutionRequest;
import com.appsmith.external.models.ActionExecutionResult;
@ -15,18 +17,16 @@ import com.appsmith.external.models.Property;
import com.appsmith.external.plugins.BasePlugin;
import com.appsmith.external.plugins.BaseRestApiPluginExecutor;
import com.appsmith.external.services.SharedConfig;
import com.appsmith.external.helpers.restApiUtils.connections.APIConnection;
import com.appsmith.external.helpers.restApiUtils.helpers.RequestCaptureFilter;
import lombok.extern.slf4j.Slf4j;
import org.pf4j.Extension;
import org.pf4j.PluginWrapper;
import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.core.publisher.Mono;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
@ -125,7 +125,7 @@ public class RestApiPlugin extends BasePlugin {
initUtils.initializeResponseWithError(errorResult);
// Set of hint messages that can be returned to the user.
Set<String> hintMessages = new HashSet();
Set<String> hintMessages = new HashSet<>();
// Initializing request URL
String url = initUtils.initializeRequestUrl(actionConfiguration, datasourceConfiguration);
@ -148,18 +148,6 @@ public class RestApiPlugin extends BasePlugin {
ActionExecutionRequest actionExecutionRequest =
RequestCaptureFilter.populateRequestFields(actionConfiguration, uri, insertedParams, objectMapper);
try {
if (uriUtils.isHostDisallowed(uri)) {
errorResult.setBody(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR.getMessage("Host not allowed."));
errorResult.setRequest(actionExecutionRequest);
return Mono.just(errorResult);
}
} catch (UnknownHostException e) {
errorResult.setBody(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR.getMessage("Unknown host."));
errorResult.setRequest(actionExecutionRequest);
return Mono.just(errorResult);
}
WebClient.Builder webClientBuilder = triggerUtils.getWebClientBuilder(actionConfiguration,
datasourceConfiguration);
String reqContentType = headerUtils.getRequestContentType(actionConfiguration, datasourceConfiguration);

12
deploy/docker/entrypoint.sh
View File

@ -128,7 +128,7 @@ init_mongodb() {
if [[ ! -f "$MONGO_DB_KEY" ]]; then
openssl rand -base64 756 > "$MONGO_DB_KEY"
fi
chmod-mongodb-key "$MONGO_DB_KEY"
use-mongodb-key "$MONGO_DB_KEY"
fi
}
@ -157,7 +157,7 @@ init_replica_set() {
mongo "127.0.0.1/appsmith" /appsmith-stacks/configuration/mongo-init.js
echo "Enabling Replica Set"
mongod --dbpath "$MONGO_DB_PATH" --shutdown || true
mongod --fork --port 27017 --dbpath "$MONGO_DB_PATH" --logpath "$MONGO_LOG_PATH" --replSet mr1 --keyFile "$MONGO_DB_KEY" --bind_ip localhost
mongod --fork --port 27017 --dbpath "$MONGO_DB_PATH" --logpath "$MONGO_LOG_PATH" --replSet mr1 --keyFile /mongodb-key --bind_ip localhost
echo "Waiting 10s for MongoDB to start with Replica Set"
sleep 10
mongo "$APPSMITH_MONGODB_URI" --eval 'rs.initiate()'
@ -180,8 +180,12 @@ init_replica_set() {
fi
}
chmod-mongodb-key() {
chmod 600 "$1"
use-mongodb-key() {
# This is a little weird. We copy the MongoDB key file to `/mongodb-key`, so that we can reliably set its permissions to 600.
# What affects the reliability of this? When the host machine of this Docker container is Windows, file permissions cannot be set on files in volumes.
# So the key file should be somewhere inside the container, and not in a volume.
cp -v "$1" /mongodb-key
chmod 600 /mongodb-key
}
# Keep Let's Encrypt directory persistent

2
deploy/docker/templates/supervisord/mongodb.conf
View File

@ -1,6 +1,6 @@
[program:mongodb]
directory=/appsmith-stacks/data/mongodb
command=mongod --port 27017 --dbpath . --logpath /appsmith-stacks/logs/%(program_name)s/db.log --replSet mr1 --keyFile key --bind_ip localhost
command=mongod --port 27017 --dbpath . --logpath /appsmith-stacks/logs/%(program_name)s/db.log --replSet mr1 --keyFile /mongodb-key --bind_ip localhost
priority=10
autostart=true
autorestart=true