Snow 1944643 network errors retry

ci/container/test_component.sh

@@ -92,6 +92,8 @@ else
     )
 fi
 
+echo "[INFO] Mocha CMD: $MOCHA_CMD"
+
 if [[ -z "$GITHUB_ACTIONS" ]]; then
     echo "[INFO] Running Internal Tests. Test result: $WORKSPACE/junit-system-test.xml"
     if ! ${MOCHA_CMD[@]} "$SOURCE_ROOT/system_test/**/*.{js,ts}"; then

lib/connection/connection_config.js

@@ -1129,15 +1129,18 @@ function createParameters() {
     {
       name: PARAM_RETRY_SF_MAX_NUM_RETRIES,
       defaultValue: 1000,
+      external: true,
       validate: isNonNegativeInteger
     },
     {
       name: PARAM_RETRY_SF_STARTING_SLEEP_TIME,
       defaultValue: 1,
+      external: true,
       validate: isNonNegativeNumber
     },
     {
       name: PARAM_RETRY_SF_MAX_SLEEP_TIME,
+      external: true,
       defaultValue: 16,
       validate: isNonNegativeNumber
     }

lib/connection/statement.js

@@ -1492,9 +1492,10 @@ function sendSfRequest(statementContext, options, appendQueryParamOnRetry) {
     // if we haven't exceeded the maximum number of retries yet and the server
     // came back with a retryable error code
     if (numRetries < maxNumRetries &&
-      err && Util.isRetryableHttpError(
-      err.response, false // no retry for HTTP 403
-    )) {
+      err && (!err.response || (Util.isRetryableHttpError(err.response, false))
+    // || Util.isNetworkError(err)
+    )
+    ) {
       // increment the retry count
       numRetries++;
       lastStatusCodeForRetry = err.response ? err.response.statusCode : 0;

test/authentication/connectionParameters.js

@@ -191,6 +191,17 @@ const oauthOktaAuthorizationCode =
       enableExperimentalAuthentication: true
     };
 
+const wiremock =
+  {
+    ...baseParameters,
+    accessUrl: null,
+    username: 'MOCK_USERNAME',
+    account: 'MOCK_ACCOUNT_NAME',
+    host: 'localhost',
+    protocol: 'http',
+    authenticator: 'OAUTH',
+  };
+
 exports.externalBrowser = externalBrowser;
 exports.okta = okta;
 exports.oauth = oauth;
@@ -220,3 +231,4 @@ exports.snowflakeAuthTestOauthOktaPassword = snowflakeAuthTestOauthOktaPassword;
 exports.snowflakeAuthTestOauthOktaClientId = snowflakeAuthTestOauthOktaClientId;
 exports.oauthSnowflakeWildcardsAuthorizationCode = oauthSnowflakeWildcardsAuthorizationCode;
 exports.oauthOktaAuthorizationCode = oauthOktaAuthorizationCode;
+exports.wiremock = wiremock;

test/integration/testConnectionRetries.js

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved.
+ */
+
+const { runWireMockAsync, addWireMockMappingsFromFile  } = require('../wiremockRunner');
+const connParameters = require('../authentication/connectionParameters');
+const testUtil = require('../integration/testUtil');
+const snowflake = require('../../lib/snowflake');
+const assert = require('assert');
+
+describe('Connection test', function () {
+  this.timeout(500000);
+  let port;
+  let wireMock;
+
+  before(async () => {
+    port = await testUtil.getFreePort();
+    wireMock = await runWireMockAsync(port);
+    snowflake.configure({
+      logLevel: 'DEBUG',
+      disableOCSPChecks: true
+    });
+  });
+
+  afterEach(async () => {
+    wireMock.scenarios.resetAllScenarios();
+  });
+
+  after(async () => {
+    await wireMock.global.shutdown();
+  });
+
+  it('Test retries after connection reset - success', async function () {
+    await addWireMockMappingsFromFile(wireMock, 'wiremock/mappings/six_reset_connection_and_correct_response.json');
+    const connectionOption = { ...connParameters.wiremock, password: 'MOCK_TOKEN', port: port, sfRetryMaxSleepTime: 2, sfRetryMaxNumRetries: 10 };
+    const connection = testUtil.createConnection(connectionOption);
+    await testUtil.connectAsync(connection);
+    await assert.doesNotReject(async () => await testUtil.executeCmdAsync(connection, ' Select 1'));
+  });
+
+  it('Test retries after malformed response', async function () {
+    await addWireMockMappingsFromFile(wireMock, 'wiremock/mappings/six_malformed_and_correct.json');
+    const connectionOption = { ...connParameters.wiremock, password: 'MOCK_TOKEN', port: port, sfRetryMaxSleepTime: 2 };
+    const connection = testUtil.createConnection(connectionOption);
+    await testUtil.connectAsync(connection);
+    await assert.doesNotReject(async () => await testUtil.executeCmdAsync(connection, ' Select 1'));
+  });
+
+  it('Test retries after connection reset - fail', async function () {
+    await addWireMockMappingsFromFile(wireMock, 'wiremock/mappings/six_reset_connection_and_correct_response.json');
+    const connectionOption = { ...connParameters.wiremock, password: 'MOCK_TOKEN', port: port, sfRetryMaxNumRetries: 1, sfRetryMaxSleepTime: 2 };
+    const connection = testUtil.createConnection(connectionOption);
+    await testUtil.connectAsync(connection);
+    await assert.rejects(
+      testUtil.executeCmdAsync(connection, ' Select 1'),
+      (err) => {
+        assert.match(err.message, /Network error. Could not reach Snowflake./);
+        return true;
+      },
+    );
+  });
+});
+
+

test/integration/wiremock/testWiremockRunner.js

@@ -1,25 +1,15 @@
 const assert = require('assert');
 const fs = require('fs');
-const net = require('net');
 const axios = require('axios');
 const { runWireMockAsync } = require('../../wiremockRunner');
 const os = require('os');
-
-async function getFreePort() {
-  return new Promise(res => {
-    const srv = net.createServer();
-    srv.listen(0, () => {
-      const port = srv.address().port;
-      srv.close(() => res(port));
-    });
-  });
-}
+const testUtil = require('../testUtil');
 
 if (os.platform !== 'win32')  {
   describe('Wiremock test', function () {
     let port, wireMock;
     before(async () => {
-      port = await getFreePort();
+      port = await testUtil.getFreePort();
       wireMock = await runWireMockAsync(port);
     });
     after(async () => {