Add retry sleep, refactor retry control flow

Author: adriandole

src/libmongoc/src/mongoc/mongoc-crypt.c

@@ -527,6 +527,27 @@ _get_stream (const char *endpoint, int32_t connecttimeoutms, const mongoc_ssl_op
    return tls_stream;
 }
 
+static int64_t
+backoff_time_usec (int64_t attempts)
+{
+   static bool seeded = false;
+   if (!seeded) {
+      srand ((uint32_t) time (NULL));
+      seeded = true;
+   }
+
+   /* Exponential backoff with jitter. */
+   const int64_t base = 200000;  /* 0.2 seconds */
+   const int64_t max = 20000000; /* 20 seconds */
+   int64_t backoff = base * (1LL << (attempts - 1));
+   if (backoff > max) {
+      backoff = max;
+   }
+
+   /* Full jitter: between 1 and current max */
+   return (int64_t) ((double) rand () / (double) RAND_MAX * (double) backoff) + 1;
+}
+
 static bool
 _state_need_kms (_state_machine_t *state_machine, bson_error_t *error)
 {
@@ -575,94 +596,93 @@ _state_need_kms (_state_machine_t *state_machine, bson_error_t *error)
       }
 
    retry:
-      while (true) {
-         mongoc_stream_destroy (tls_stream);
-         tls_stream = _get_stream (endpoint, sockettimeout, ssl_opt, error);
+      if (retry_count > 0) {
+         int64_t sleep_usec = backoff_time_usec (retry_count);
+         _mongoc_usleep (sleep_usec);
+      }
+
+      mongoc_stream_destroy (tls_stream);
+      tls_stream = _get_stream (endpoint, sockettimeout, ssl_opt, error);
 #ifdef MONGOC_ENABLE_SSL_SECURE_CHANNEL
-         /* Retry once with schannel as a workaround for CDRIVER-3566. */
-         if (!tls_stream) {
-            tls_stream = _get_stream (endpoint, sockettimeout, ssl_opt, error);
-         }
+      /* Retry once with schannel as a workaround for CDRIVER-3566. */
+      if (!tls_stream) {
+         tls_stream = _get_stream (endpoint, sockettimeout, ssl_opt, error);
+      }
 #endif
-         if (!tls_stream) {
-            retry_count++;
-            if (retry_count < max_tcp_retries) {
-               /* Always safe to retry stream creation. */
-               continue;
-            } else {
-               /* TLS errors are set in _get_stream */
-               goto fail;
-            }
+      if (!tls_stream) {
+         retry_count++;
+         if (retry_count < max_tcp_retries) {
+            /* Always safe to retry stream creation. */
+            goto retry;
+         } else {
+            /* TLS errors are set in _get_stream */
+            goto fail;
          }
+      }
 
-         iov.iov_base = (char *) mongocrypt_binary_data (http_req);
-         iov.iov_len = mongocrypt_binary_len (http_req);
+      iov.iov_base = (char *) mongocrypt_binary_data (http_req);
+      iov.iov_len = mongocrypt_binary_len (http_req);
 
-         if (!_mongoc_stream_writev_full (tls_stream, &iov, 1, sockettimeout, error)) {
-            retry_count++;
-            if (retry_count < max_tcp_retries) {
-               /* Always safe to retry unsuccessful writes. */
-               continue;
-            } else {
-               bson_set_error (error,
-                               MONGOC_ERROR_STREAM,
-                               MONGOC_ERROR_STREAM_SOCKET,
-                               "Failed to write to KMS stream: %s",
-                               endpoint);
-               goto fail;
-            }
+      if (!_mongoc_stream_writev_full (tls_stream, &iov, 1, sockettimeout, error)) {
+         retry_count++;
+         if (retry_count < max_tcp_retries && mongocrypt_kms_ctx_fail (kms_ctx)) {
+            goto retry;
+         } else {
+            bson_set_error (
+               error, MONGOC_ERROR_STREAM, MONGOC_ERROR_STREAM_SOCKET, "Failed to write to KMS stream: %s", endpoint);
+            goto fail;
          }
+      }
 
-         /* Read and feed reply. */
-         while (mongocrypt_kms_ctx_bytes_needed (kms_ctx) > 0) {
+      /* Read and feed reply. */
+      while (mongocrypt_kms_ctx_bytes_needed (kms_ctx) > 0) {
 #define BUFFER_SIZE 1024
-            uint8_t buf[BUFFER_SIZE];
-            uint32_t bytes_needed = mongocrypt_kms_ctx_bytes_needed (kms_ctx);
-            ssize_t read_ret;
+         uint8_t buf[BUFFER_SIZE];
+         uint32_t bytes_needed = mongocrypt_kms_ctx_bytes_needed (kms_ctx);
+         ssize_t read_ret;
 
-            int64_t sleep_usec = mongocrypt_kms_ctx_usleep (kms_ctx);
-            if (sleep_usec > 0) {
-               _mongoc_usleep (sleep_usec);
-            }
+         int64_t sleep_usec = mongocrypt_kms_ctx_usleep (kms_ctx);
+         if (sleep_usec > 0) {
+            _mongoc_usleep (sleep_usec);
+         }
 
-            /* Cap the bytes requested at the buffer size. */
-            if (bytes_needed > BUFFER_SIZE) {
-               bytes_needed = BUFFER_SIZE;
-            }
+         /* Cap the bytes requested at the buffer size. */
+         if (bytes_needed > BUFFER_SIZE) {
+            bytes_needed = BUFFER_SIZE;
+         }
 
-            read_ret = mongoc_stream_read (tls_stream, buf, bytes_needed, 1 /* min_bytes. */, sockettimeout);
-            if (read_ret <= 0) {
-               retry_count++;
-               if (retry_count < max_tcp_retries && mongocrypt_kms_ctx_fail (kms_ctx)) {
-                  goto retry;
+         read_ret = mongoc_stream_read (tls_stream, buf, bytes_needed, 1 /* min_bytes. */, sockettimeout);
+         if (read_ret <= 0) {
+            retry_count++;
+            if (retry_count < max_tcp_retries && mongocrypt_kms_ctx_fail (kms_ctx)) {
+               goto retry;
+            } else {
+               if (read_ret == -1) {
+                  bson_set_error (error,
+                                  MONGOC_ERROR_STREAM,
+                                  MONGOC_ERROR_STREAM_SOCKET,
+                                  "failed to read from KMS stream: %d",
+                                  errno);
+                  goto fail;
                } else {
-                  if (read_ret == -1) {
-                     bson_set_error (error,
-                                     MONGOC_ERROR_STREAM,
-                                     MONGOC_ERROR_STREAM_SOCKET,
-                                     "failed to read from KMS stream: %d",
-                                     errno);
-                     goto fail;
-                  } else {
-                     bson_set_error (
-                        error, MONGOC_ERROR_STREAM, MONGOC_ERROR_STREAM_SOCKET, "unexpected EOF from KMS stream");
-                     goto fail;
-                  }
+                  bson_set_error (
+                     error, MONGOC_ERROR_STREAM, MONGOC_ERROR_STREAM_SOCKET, "unexpected EOF from KMS stream");
+                  goto fail;
                }
             }
-            mongocrypt_binary_destroy (http_reply);
+         }
+         mongocrypt_binary_destroy (http_reply);
 
-            BSON_ASSERT (bson_in_range_signed (uint32_t, read_ret));
-            http_reply = mongocrypt_binary_new_from_data (buf, (uint32_t) read_ret);
-            if (!mongocrypt_kms_ctx_feed (kms_ctx, http_reply)) {
-               _kms_ctx_check_error (kms_ctx, error, true);
-               goto fail;
-            }
+         BSON_ASSERT (bson_in_range_signed (uint32_t, read_ret));
+         http_reply = mongocrypt_binary_new_from_data (buf, (uint32_t) read_ret);
+         if (!mongocrypt_kms_ctx_feed (kms_ctx, http_reply)) {
+            _kms_ctx_check_error (kms_ctx, error, true);
+            goto fail;
          }
-         kms_ctx = mongocrypt_ctx_next_kms_ctx (state_machine->ctx);
-         retry_count = 0;
-         break;
       }
+      kms_ctx = mongocrypt_ctx_next_kms_ctx (state_machine->ctx);
+      retry_count = 0;
+      break;
    }
    /* When NULL is returned by mongocrypt_ctx_next_kms_ctx, this can either be
     * an error or end-of-list. */