Retry when encountering a network error establishing an initial connection to a server

eramongodb · eramongodb · commit dca28b07a7a2 · 2022-11-04T10:17:25.000-05:00
diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c
@@ -2362,9 +2362,6 @@ mongoc_cluster_stream_for_server (mongoc_cluster_t *cluster,
    BSON_ASSERT (reply || true);
    BSON_ASSERT (error || true);
 
-   mongoc_server_stream_t *server_stream = NULL;
-   bson_error_t err_local = {0};
-
    ENTRY;
 
    BSON_ASSERT (cluster);
@@ -2378,12 +2375,10 @@ mongoc_cluster_stream_for_server (mongoc_cluster_t *cluster,
       RETURN (NULL);
    }
 
-   if (!error) {
-      error = &err_local;
-   }
 
-   server_stream = _mongoc_cluster_stream_for_server (
-      cluster, server_id, reconnect_ok, cs, reply, error);
+   mongoc_server_stream_t *const server_stream =
+      _mongoc_cluster_stream_for_server (
+         cluster, server_id, reconnect_ok, cs, reply, error);
 
    if (_in_sharded_txn (cs)) {
       _mongoc_client_session_pin (cs, server_id);
@@ -2801,6 +2796,7 @@ _mongoc_cluster_stream_for_optype (mongoc_cluster_t *cluster,
                                    mongoc_ss_optype_t optype,
                                    const mongoc_read_prefs_t *read_prefs,
                                    mongoc_client_session_t *cs,
+                                   bool is_retryable,
                                    bson_t *reply,
                                    bson_error_t *error)
 {
@@ -2844,14 +2840,73 @@ _mongoc_cluster_stream_for_optype (mongoc_cluster_t *cluster,
       }
    }
 
-   /* connect or reconnect to server if necessary */
-   server_stream = _mongoc_cluster_stream_for_server (
-      cluster, server_id, true /* reconnect_ok */, cs, reply, error);
+   bson_t first_reply;
+   bson_error_t first_error = {0};
+
+   server_stream = _mongoc_cluster_stream_for_server (cluster,
+                                                      server_id,
+                                                      true /* reconnect_ok */,
+                                                      cs,
+                                                      &first_reply,
+                                                      &first_error);
+
    if (server_stream) {
       server_stream->must_use_primary = must_use_primary;
+      RETURN (server_stream);
    }
 
-   RETURN (server_stream);
+   // Important: authentication errors are also considered retryable even if
+   // they not considered a network error.
+   const bool retryable_error = _mongoc_error_is_network (&first_error) ||
+                                _mongoc_error_is_auth (&first_error);
+
+   if (is_retryable && retryable_error) {
+      bson_t retry_reply;
+      bson_error_t retry_error = {0};
+
+      server_stream =
+         _mongoc_cluster_stream_for_server (cluster,
+                                            server_id,
+                                            true /* reconnect_ok */,
+                                            cs,
+                                            &retry_reply,
+                                            &retry_error);
+
+      if (server_stream) {
+         server_stream->must_use_primary = must_use_primary;
+         server_stream->retry_attempted = true;
+         bson_destroy (&first_reply);
+         RETURN (server_stream);
+      }
+
+      if (optype != MONGOC_SS_READ) {
+         // Retryable Writes Spec: When the driver encounters a network error
+         // establishing an initial connection to a server, it MUST add a
+         // RetryableWriteError label to that error if the MongoClient
+         // performing the operation has the retryWrites configuration option
+         // set to true.
+         _mongoc_write_error_append_retryable_label (&first_reply);
+      }
+
+      bson_destroy (&retry_reply);
+   }
+
+   // Retryable Writes Spec: If the driver cannot select a server for the retry
+   // attempt [...], retrying is not possible and drivers MUST raise the
+   // original retryable error.
+   {
+      if (reply) {
+         bson_steal (reply, &first_reply);
+      } else {
+         bson_destroy (&first_reply);
+      }
+
+      if (error) {
+         *error = first_error;
+      }
+   }
+
+   RETURN (NULL);
 }
 
 mongoc_server_stream_t *
@@ -2864,8 +2919,14 @@ mongoc_cluster_stream_for_reads (mongoc_cluster_t *cluster,
    const mongoc_read_prefs_t *const prefs_override =
       _mongoc_client_session_in_txn (cs) ? cs->txn.opts.read_prefs : read_prefs;
 
+   // Retryable Reads Spec: This boolean option determines whether retryable
+   // behavior will be applied to all read operations executed within the
+   // MongoClient.
+   const bool is_retryable = mongoc_uri_get_option_as_bool (
+      cluster->uri, MONGOC_URI_RETRYREADS, MONGOC_DEFAULT_RETRYREADS);
+
    return _mongoc_cluster_stream_for_optype (
-      cluster, MONGOC_SS_READ, prefs_override, cs, reply, error);
+      cluster, MONGOC_SS_READ, prefs_override, cs, is_retryable, reply, error);
 }
 
 mongoc_server_stream_t *
@@ -2874,8 +2935,11 @@ mongoc_cluster_stream_for_writes (mongoc_cluster_t *cluster,
                                   bson_t *reply,
                                   bson_error_t *error)
 {
+   const bool is_retryable = mongoc_uri_get_option_as_bool (
+      cluster->uri, MONGOC_URI_RETRYWRITES, MONGOC_DEFAULT_RETRYWRITES);
+
    return _mongoc_cluster_stream_for_optype (
-      cluster, MONGOC_SS_WRITE, NULL, cs, reply, error);
+      cluster, MONGOC_SS_WRITE, NULL, cs, is_retryable, reply, error);
 }
 
 mongoc_server_stream_t *
@@ -2889,10 +2953,14 @@ mongoc_cluster_stream_for_aggr_with_write (
    const mongoc_read_prefs_t *const prefs_override =
       _mongoc_client_session_in_txn (cs) ? cs->txn.opts.read_prefs : read_prefs;
 
+   const bool is_retryable = mongoc_uri_get_option_as_bool (
+      cluster->uri, MONGOC_URI_RETRYWRITES, MONGOC_DEFAULT_RETRYWRITES);
+
    return _mongoc_cluster_stream_for_optype (cluster,
                                              MONGOC_SS_AGGREGATE_WITH_WRITE,
                                              prefs_override,
                                              cs,
+                                             is_retryable,
                                              reply,
                                              error);
 }
diff --git a/src/libmongoc/src/mongoc/mongoc-cmd.c b/src/libmongoc/src/mongoc/mongoc-cmd.c
@@ -707,6 +707,10 @@ _allow_txn_number (const mongoc_cmd_parts_t *parts,
       return false;
    }
 
+   if (server_stream->retry_attempted) {
+      return false;
+   }
+
    if (server_stream->sd->max_wire_version < WIRE_VERSION_RETRY_WRITES) {
       return false;
    }
@@ -740,6 +744,10 @@ _is_retryable_write (const mongoc_cmd_parts_t *parts,
       return false;
    }
 
+   if (server_stream->retry_attempted) {
+      return false;
+   }
+
    if (server_stream->sd->max_wire_version < WIRE_VERSION_RETRY_WRITES) {
       return false;
    }
@@ -777,6 +785,10 @@ _is_retryable_read (const mongoc_cmd_parts_t *parts,
       return false;
    }
 
+   if (server_stream->retry_attempted) {
+      return false;
+   }
+
    if (server_stream->sd->max_wire_version < WIRE_VERSION_RETRY_READS) {
       return false;
    }
diff --git a/src/libmongoc/src/mongoc/mongoc-server-stream-private.h b/src/libmongoc/src/mongoc/mongoc-server-stream-private.h
@@ -31,12 +31,16 @@ BSON_BEGIN_DECLS
 
 typedef struct _mongoc_server_stream_t {
    mongoc_topology_description_type_t topology_type;
-   mongoc_server_description_t *sd; /* owned */
-   bson_t cluster_time;             /* owned */
-   mongoc_stream_t *stream;         /* borrowed */
-   /** If the stream was created in a way that may have overwritten the user's
-    * readPreference, we need to know if server selection forced that change. */
+   mongoc_server_description_t *sd; // owned
+   bson_t cluster_time;             // owned
+   mongoc_stream_t *stream;         // borrowed
+   // If the stream was created in a way that may have overwritten the user's
+   // readPreference, we need to know if server selection forced that change.
    bool must_use_primary;
+   // True if this server stream was acquired during a retry attempt triggered
+   // by a network error establishing an initial connection. Used to avoid
+   // further retry attempts.
+   bool retry_attempted;
 } mongoc_server_stream_t;
 
 
diff --git a/src/libmongoc/src/mongoc/mongoc-server-stream.c b/src/libmongoc/src/mongoc/mongoc-server-stream.c
@@ -38,6 +38,7 @@ mongoc_server_stream_new (const mongoc_topology_description_t *td,
    server_stream->sd = sd;         /* becomes owned */
    server_stream->stream = stream; /* merely borrowed */
    server_stream->must_use_primary = false;
+   server_stream->retry_attempted = false;
 
    return server_stream;
 }

Original file line number	Diff line number	Diff line change
`@@ -707,6 +707,10 @@ _allow_txn_number (const mongoc_cmd_parts_t *parts,`
`707`	`707`	`return false;`
`708`	`708`	`}`
`709`	`709`
	`710`	`+ if (server_stream->retry_attempted) {`
	`711`	`+ return false;`
	`712`	`+ }`
	`713`	`+`
`710`	`714`	`if (server_stream->sd->max_wire_version < WIRE_VERSION_RETRY_WRITES) {`
`711`	`715`	`return false;`
`712`	`716`	`}`
`@@ -740,6 +744,10 @@ _is_retryable_write (const mongoc_cmd_parts_t *parts,`
`740`	`744`	`return false;`
`741`	`745`	`}`
`742`	`746`
	`747`	`+ if (server_stream->retry_attempted) {`
	`748`	`+ return false;`
	`749`	`+ }`
	`750`	`+`
`743`	`751`	`if (server_stream->sd->max_wire_version < WIRE_VERSION_RETRY_WRITES) {`
`744`	`752`	`return false;`
`745`	`753`	`}`
`@@ -777,6 +785,10 @@ _is_retryable_read (const mongoc_cmd_parts_t *parts,`
`777`	`785`	`return false;`
`778`	`786`	`}`
`779`	`787`
	`788`	`+ if (server_stream->retry_attempted) {`
	`789`	`+ return false;`
	`790`	`+ }`
	`791`	`+`
`780`	`792`	`if (server_stream->sd->max_wire_version < WIRE_VERSION_RETRY_READS) {`
`781`	`793`	`return false;`
`782`	`794`	`}`
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,7 @@ mongoc_server_stream_new (const mongoc_topology_description_t *td,`
`38`	`38`	`server_stream->sd = sd; /* becomes owned */`
`39`	`39`	`server_stream->stream = stream; /* merely borrowed */`
`40`	`40`	`server_stream->must_use_primary = false;`
	`41`	`+ server_stream->retry_attempted = false;`
`41`	`42`
`42`	`43`	`return server_stream;`
`43`	`44`	`}`