Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions app/server/appsmith-plugins/mssqlPlugin/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,12 @@
</exclusions>
</dependency>

<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>

</dependencies>

<build>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;

import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY;
Expand Down Expand Up @@ -268,9 +270,21 @@ public Mono<ActionExecutionResult> executeCommon(
MessageFormat.format("Before executing Mssql query [{0}]", query), hikariDSConnection);
;

// The reactive timeout below cannot interrupt this blocking call: it reports the timeout only
// after the call returns. So the query gets the same timeout here:
// - the driver's query timeout, while SQL Server has not answered yet;
// - a timer that cancels the query at the deadline, while a read of later results blocks;
// - checks between reads, which stop a query that returns results without end.
final int timeoutMillis = actionConfiguration.getTimeoutInMillisecond();
final int queryTimeoutSeconds = MssqlExecuteUtils.queryTimeoutSeconds(timeoutMillis);
final long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
ScheduledFuture<?> cancelAtDeadline = null;

try {
if (FALSE.equals(preparedStatement)) {
statement = sqlConnectionFromPool.createStatement();
statement.setQueryTimeout(queryTimeoutSeconds);
cancelAtDeadline = MssqlExecuteUtils.cancelAtDeadline(statement, deadlineNanos);
isResultSet = statement.execute(query);
resultSet = statement.getResultSet();
} else {
Expand All @@ -289,6 +303,8 @@ public Mono<ActionExecutionResult> executeCommon(
parameters.get(i).getKey(),
parameters.get(i).getValue())));

preparedQuery.setQueryTimeout(queryTimeoutSeconds);
cancelAtDeadline = MssqlExecuteUtils.cancelAtDeadline(preparedQuery, deadlineNanos);
isResultSet = preparedQuery.execute();
resultSet = preparedQuery.getResultSet();
}
Expand All @@ -300,16 +316,28 @@ public Mono<ActionExecutionResult> executeCommon(
isResultSet,
preparedStatement,
statement,
preparedQuery);
preparedQuery,
timeoutMillis,
deadlineNanos);

} catch (SQLException e) {
// After the deadline, the error comes from the cancel or the driver's query timeout.
String errorMessage = MssqlExecuteUtils.isPastDeadline(deadlineNanos)
? String.format(MssqlErrorMessages.QUERY_TIMEOUT_ERROR_MSG, timeoutMillis)
: e.getMessage();
return Mono.error(new AppsmithPluginException(
MssqlPluginError.QUERY_EXECUTION_FAILED,
MssqlErrorMessages.QUERY_EXECUTION_FAILED_ERROR_MSG,
e.getMessage(),
errorMessage,
"SQLSTATE: " + e.getSQLState()));

} finally {
// Stop the timer before the connection goes back to the pool, so that it never cancels
// a statement on a connection another query is using.
if (cancelAtDeadline != null) {
cancelAtDeadline.cancel(false);
}

// Log HikariCP status
logHikariCPStatus(
MessageFormat.format("After executing Mssql query [{0}]", query),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ public class MssqlErrorMessages extends BasePluginErrorMessages {
public static final String QUERY_EXECUTION_FAILED_ERROR_MSG =
"Your query failed to execute. Please check more information in the error details.";

public static final String QUERY_TIMEOUT_ERROR_MSG = "The query ran longer than its timeout of %d ms, so Appsmith"
+ " stopped it. Statements in the query that already ran may have been saved.";

public static final String QUERY_PREPARATION_FAILED_ERROR_MSG = "Query preparation failed while inserting value: %s"
+ " for binding: {{%s}}. Please check the query again.";

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.external.plugins.utils;

import com.external.plugins.exceptions.MssqlErrorMessages;
import microsoft.sql.DateTimeOffset;
import org.apache.commons.lang3.ObjectUtils;

Expand All @@ -8,6 +9,7 @@
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.SQLTimeoutException;
import java.sql.Statement;
import java.text.MessageFormat;
import java.time.OffsetDateTime;
Expand All @@ -18,6 +20,9 @@
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import static com.appsmith.external.helpers.PluginUtils.getColumnsListForJdbcPlugin;
import static com.appsmith.external.helpers.PluginUtils.safelyCloseSingleConnectionFromHikariCP;
Expand Down Expand Up @@ -82,9 +87,13 @@ public static void populateRowsAndColumns(
boolean isResultSet,
Boolean preparedStatement,
Statement statement,
PreparedStatement preparedQuery)
PreparedStatement preparedQuery,
int timeoutMillis,
long deadlineNanos)
throws SQLException {

Statement executed = FALSE.equals(preparedStatement) ? statement : preparedQuery;

if (!isResultSet) {
Object updateCount = FALSE.equals(preparedStatement)
? ObjectUtils.defaultIfNull(statement.getUpdateCount(), 0)
Expand All @@ -97,6 +106,8 @@ public static void populateRowsAndColumns(
columnsList.addAll(getColumnsListForJdbcPlugin(metaData));

while (resultSet.next()) {
stopAtDeadline(executed, timeoutMillis, deadlineNanos);

// Use `LinkedHashMap` here so that the column ordering is preserved in the response.
Map<String, Object> row = new LinkedHashMap<>(colCount);

Expand Down Expand Up @@ -150,7 +161,108 @@ public static void populateRowsAndColumns(
}
}

drainRemainingResults(FALSE.equals(preparedStatement) ? statement : preparedQuery);
drainRemainingResults(executed, timeoutMillis, deadlineNanos);
}

/**
* Returns the JDBC query timeout, in whole seconds, for an action timeout in milliseconds. Rounds up, so the
* driver never stops a query before the action's own timeout. JDBC treats 0 as no timeout, so the result is at
* least 1.
*/
public static int queryTimeoutSeconds(int timeoutMillis) {
return (int) Math.max(1, (timeoutMillis + 999L) / 1000);
}

/** Returns true when the action's timeout has passed. */
public static boolean isPastDeadline(long deadlineNanos) {
return System.nanoTime() - deadlineNanos >= 0;
}

/**
* Cancels the statement on SQL Server at the deadline, from another thread. A read of the results can block
* inside the driver, for example while a later statement runs {@code WAITFOR}, and the driver's own query
* timeout no longer runs once the first response has arrived. The checks between reads cannot interrupt such a
* read. Cancelling ends it with an error.
*
* <p>The caller must cancel the returned task before it closes the statement and returns the connection to the
* pool.
*/
public static ScheduledFuture<?> cancelAtDeadline(Statement statement, long deadlineNanos) {
long delayNanos = Math.max(0, deadlineNanos - System.nanoTime());
// The timer thread only starts the cancel. Sending it to a stalled server can block, and that must not
// delay the cancels of other queries.
return CANCEL_TIMER.schedule(
() -> CANCEL_THREADS.start(() -> {
try {
if (!statement.isClosed()) {
statement.cancel();
}
} catch (SQLException e) {
// The statement finished or was closed meanwhile. Nothing is left to cancel.
}
}),
delayNanos,
TimeUnit.NANOSECONDS);
}

/**
* One daemon thread that starts the cancels at their deadlines. A task stopped before its deadline leaves the
* queue at once, so finished queries do not pile up in it.
*/
private static final ScheduledThreadPoolExecutor CANCEL_TIMER = newCancelTimer();

/** Each cancel runs on its own virtual thread. */
private static final Thread.Builder CANCEL_THREADS = Thread.ofVirtual().name("mssql-query-cancel-", 0);

private static ScheduledThreadPoolExecutor newCancelTimer() {
ScheduledThreadPoolExecutor timer = new ScheduledThreadPoolExecutor(1, task -> {
Thread thread = new Thread(task, "mssql-query-cancel-timer");
thread.setDaemon(true);
return thread;
});
timer.setRemoveOnCancelPolicy(true);
return timer;
}

/** The number of cancels waiting for their deadline. For tests. */
static int pendingCancels() {
return CANCEL_TIMER.getQueue().size();
}

/** Fails the query with a timeout when the action's timeout has passed. */
private static void stopAtDeadline(Statement statement, int timeoutMillis, long deadlineNanos) throws SQLException {
if (isPastDeadline(deadlineNanos)) {
throw cancelForTimeout(statement, timeoutMillis, null);
}
}

/**
* Cancels the query on SQL Server and returns the timeout error. Closing the statement afterwards would otherwise
* read every remaining result, which can go on without end.
*
* @param cause the error the driver raised after the deadline, or null
*/
private static DeadlineReached cancelForTimeout(Statement statement, int timeoutMillis, SQLException cause) {
DeadlineReached timeout = new DeadlineReached(timeoutMillis);
if (cause != null) {
timeout.initCause(cause);
}
if (statement != null) {
try {
statement.cancel();
} catch (SQLException e) {
// Still report the timeout.
timeout.addSuppressed(e);
}
}
return timeout;
}

/** The plugin's own timeout error, so that it is not reported as a failed statement. */
private static final class DeadlineReached extends SQLTimeoutException {
private DeadlineReached(int timeoutMillis) {
super(String.format(MssqlErrorMessages.QUERY_TIMEOUT_ERROR_MSG, timeoutMillis));
}
}

/**
Expand All @@ -159,27 +271,39 @@ public static void populateRowsAndColumns(
* {@code UPDATE ...; INSERT INTO missing_table ...} reported success. Other statements may already have run: the
* ones before the error, and, because SQL Server continues after many statement errors, some after it. Without an
* explicit transaction they are committed. So the error says so.
*
* <p>A query can return results without end, for example {@code WHILE 1 = 1 SELECT 1}. So the reading stops at
* the action's timeout: the query is cancelled on SQL Server and fails with a timeout error.
*/
private static void drainRemainingResults(Statement statement) throws SQLException {
private static void drainRemainingResults(Statement statement, int timeoutMillis, long deadlineNanos)
throws SQLException {
if (statement == null) {
return;
}

try {
while (true) {
stopAtDeadline(statement, timeoutMillis, deadlineNanos);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if (statement.getMoreResults()) {
// A later SELECT can fail while its rows are read, for example on a divide by zero in row 5.
// Step through its rows (they are not returned) so that error surfaces too.
try (ResultSet laterRows = statement.getResultSet()) {
while (laterRows != null && laterRows.next()) {
// Nothing to keep.
stopAtDeadline(statement, timeoutMillis, deadlineNanos);
}
}
} else if (statement.getUpdateCount() == -1) {
break;
}
}
} catch (DeadlineReached e) {
throw e;
} catch (SQLException e) {
if (isPastDeadline(deadlineNanos)) {
// A cancel at the deadline, or any other error after it. Some errors do not end the query, so
// cancel it before reporting the timeout.
throw cancelForTimeout(statement, timeoutMillis, e);
}
throw new SQLException(
"A statement in this query failed. Other statements in the query may already have run."
+ (e.getMessage() == null ? "" : " " + e.getMessage()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@
import org.testcontainers.containers.MSSQLServerContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import reactor.test.StepVerifier;

import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
Expand All @@ -36,6 +38,8 @@
import static com.external.plugins.MssqlTestDBContainerManager.createDatasourceConfiguration;
import static com.external.plugins.MssqlTestDBContainerManager.mssqlPluginExecutor;
import static com.external.plugins.MssqlTestDBContainerManager.runSQLQueryOnMssqlTestDB;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
Expand Down Expand Up @@ -497,4 +501,74 @@ public void connectionUrl_turnsOffTheNativeVectorType() {
connectionPool.close();
}
}

private static Stream<Arguments> endlessLaterResults() {
return Stream.of(true, false)
.flatMap(prepared -> Stream.of(
// Endless result sets after the first.
Arguments.of(
"SELECT 1 AS first_result; WHILE 1 = 1 SELECT 1 AS later_result",
prepared,
"ran longer than its timeout of 3000 ms"),
// Endless update counts after the first.
Arguments.of(
"SELECT 1 AS first_result; WHILE 1 = 1 UPDATE driver_dml SET name = name",
prepared,
"ran longer than its timeout of 3000 ms"),
// One statement that runs longer than the timeout. The driver's query timeout, or the timer,
// stops it.
Arguments.of("WAITFOR DELAY '00:00:20'", prepared, "ran longer than its timeout of 3000 ms"),
// A later statement that waits, after a first result the server sends at once. The wait
// happens inside a read of the later results.
Arguments.of(
"SELECT 1 AS first_result; RAISERROR('flush', 0, 1) WITH NOWAIT;"
+ " WAITFOR DELAY '00:00:20'; SELECT 2 AS later_result",
prepared,
"ran longer than its timeout of 3000 ms")));
}

// AIP-FD-38: a query that runs past its timeout must not hold a pooled connection and a worker thread. The
// reactive timeout cannot interrupt the blocking JDBC call, so before the fix such a query ran on: the endless
// ones forever, because the plugin reads the later results to find errors (APP-16536).
@ParameterizedTest(name = "{0}, prepared statement {1}")
@MethodSource("endlessLaterResults")
void should_failAndFreeTheConnection_when_theQueryRunsPastItsTimeout(
String sql, boolean preparedStatement, String expectedError) {
// Given
HikariDataSource connectionPool = mssqlPluginExecutor
.datasourceCreate(createDatasourceConfiguration(container))
.block();
assertThat(connectionPool).isNotNull();
ActionConfiguration actionConfiguration = new ActionConfiguration();
actionConfiguration.setBody(sql);
actionConfiguration.setTimeoutInMillisecond("3000");
actionConfiguration.setPluginSpecifiedTemplates(
List.of(new Property("preparedStatement", String.valueOf(preparedStatement))));

try {
// When
// Before the fix, this call never returned: the reactive timeout cannot interrupt the blocking read.
Duration took = StepVerifier.create(mssqlPluginExecutor.executeParameterized(
connectionPool,
new ExecuteActionDTO(),
createDatasourceConfiguration(container),
actionConfiguration))
// Then
.assertNext(result -> {
assertThat(result.getIsExecutionSuccess()).isFalse();
assertThat(String.valueOf(result.getBody())).contains(expectedError);
})
.expectComplete()
.verify(Duration.ofSeconds(30));
assertThat(took).isLessThan(Duration.ofSeconds(10));

// The connection goes back to the pool shortly after the timeout.
await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> assertThat(
connectionPool.getHikariPoolMXBean().getActiveConnections())
.as("connections the query still holds after its timeout")
.isZero());
} finally {
connectionPool.close();
}
}
}
Loading
Loading