GODRIVER-1655 retry replSetStepDown for LockTimeout errors

Author: Divjot Arora

mongo/integration/crud_helpers_test.go

@@ -31,6 +31,7 @@ import (
 var (
 	emptyDoc                        = []byte{5, 0, 0, 0, 0}
 	errorCommandNotFound      int32 = 59
+	errorLockTimeout          int32 = 24
 	killAllSessionsErrorCodes       = map[int32]struct{}{
 		errorInterrupted:     {},
 		errorCommandNotFound: {}, // the killAllSessions command does not exist on server versions < 3.6
@@ -1406,12 +1407,38 @@ func executeAdminCommand(mt *mtest.T, op *operation) {
 		_ = client.Disconnect(mtest.Background)
 	}()
 
-	db := client.Database("admin")
 	cmd := op.Arguments.Lookup("command").Document()
+	if op.CommandName == "replSetStepDown" {
+		// replSetStepDown can fail with transient errors, so we use executeAdminCommandWithRetry to handle them and
+		// retry until a timeout is hit.
+		executeAdminCommandWithRetry(mt, client, cmd)
+		return
+	}
+
+	db := client.Database("admin")
 	err = db.RunCommand(mtest.Background, cmd).Err()
 	assert.Nil(mt, err, "RunCommand error for command %q: %v", op.CommandName, err)
 }
 
+func executeAdminCommandWithRetry(mt *mtest.T, client *mongo.Client, cmd interface{}, opts ...*options.RunCmdOptions) {
+	mt.Helper()
+
+	ctx, cancel := context.WithTimeout(mtest.Background, 10*time.Second)
+	defer cancel()
+
+	for {
+		err := client.Database("admin").RunCommand(ctx, cmd, opts...).Err()
+		if err == nil {
+			return
+		}
+
+		if ce, ok := err.(mongo.CommandError); ok && ce.Code == errorLockTimeout {
+			continue
+		}
+		mt.Fatalf("error executing command: %v", err)
+	}
+}
+
 // verification function to use for all count operations
 func verifyCountResult(mt *mtest.T, actualResult int64, expectedResult interface{}) {
 	mt.Helper()

mongo/integration/primary_stepdown_test.go

@@ -15,7 +15,6 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/integration/mtest"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"go.mongodb.org/mongo-driver/mongo/readpref"
 )
 
 const (
@@ -66,11 +65,14 @@ func TestConnectionsSurvivePrimaryStepDown(t *testing.T) {
 		defer cur.Close(mtest.Background)
 		assert.True(mt, cur.Next(mtest.Background), "expected Next true, got false")
 
-		err = mt.Client.Database("admin").RunCommand(mtest.Background, bson.D{
+		// replSetStepDown can fail with transient errors, so we use executeAdminCommandWithRetry to handle them and
+		// retry until a timeout is hit.
+		stepDownCmd := bson.D{
 			{"replSetStepDown", 5},
 			{"force", true},
-		}, options.RunCmd().SetReadPreference(readpref.Primary())).Err()
-		assert.Nil(mt, err, "replSetStepDown error: %v", err)
+		}
+		stepDownOpts := options.RunCmd().SetReadPreference(mtest.PrimaryRp)
+		executeAdminCommandWithRetry(mt, mt.Client, stepDownCmd, stepDownOpts)
 
 		assert.True(mt, cur.Next(mtest.Background), "expected Next true, got false")
 		assert.False(mt, isPoolCleared(), "expected pool to not be cleared but was")