PYTHON-2453 Add MongoDB Versioned API (#536)

Add pymongo.server_api.ServerApi and the MongoClient server_api option.
Support Unified Test Format version 1.1 (serverParameters in
runOnRequirements)
Skip dropRole tests due to SERVER-53499.

(cherry picked from commit ac2f506)

Author: ShaneHarvey

.evergreen/config.yml

@@ -290,6 +290,7 @@ functions:
             STORAGE_ENGINE=${STORAGE_ENGINE} \
             DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \
             ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
+            REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
             sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
     # run-orchestration generates expansion file with the MONGODB_URI for the cluster
     - command: expansions.update
@@ -425,6 +426,7 @@ functions:
             AUTH=${AUTH} \
             SSL=${SSL} \
             DATA_LAKE=${DATA_LAKE} \
+            MONGODB_API_VERSION=${MONGODB_API_VERSION} \
             sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   "run enterprise auth tests":
@@ -2023,6 +2025,19 @@ axes:
         variables:
           SETDEFAULTENCODING: "cp1251"
 
+  - id: requireApiVersion
+    display_name: "requireApiVersion"
+    values:
+      - id: "requireApiVersion1"
+        display_name: "requireApiVersion1"
+        tags: [ "requireApiVersion_tag" ]
+        variables:
+          # REQUIRE_API_VERSION is set to make drivers-evergreen-tools
+          # start a cluster with the requireApiVersion parameter.
+          REQUIRE_API_VERSION: "1"
+          # MONGODB_API_VERSION is the apiVersion to use in the test suite.
+          MONGODB_API_VERSION: "1"
+
 buildvariants:
 - matrix_name: "tests-all"
   matrix_spec:
@@ -2605,6 +2620,17 @@ buildvariants:
   tasks:
     - name: atlas-data-lake-tests
 
+- matrix_name: "versioned-api-tests"
+  matrix_spec:
+    platform: ubuntu-16.04
+    python-version: ["2.7", "3.9"]
+    auth: "auth"
+    requireApiVersion: "*"
+  display_name: "requireApiVersion ${python-version}"
+  tasks:
+    # Versioned API was introduced in MongoDB 4.7
+    - "test-latest-standalone"
+
 - matrix_name: "ocsp-test"
   matrix_spec:
     platform: ubuntu-16.04

.evergreen/run-tests.sh

@@ -26,6 +26,7 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK:-}
 C_EXTENSIONS=${C_EXTENSIONS:-}
 COVERAGE=${COVERAGE:-}
 COMPRESSORS=${COMPRESSORS:-}
+MONGODB_API_VERSION=${MONGODB_API_VERSION:-}
 TEST_ENCRYPTION=${TEST_ENCRYPTION:-}
 LIBMONGOCRYPT_URL=${LIBMONGOCRYPT_URL:-}
 SETDEFAULTENCODING=${SETDEFAULTENCODING:-}
@@ -35,6 +36,11 @@ if [ -n "$COMPRESSORS" ]; then
     export COMPRESSORS=$COMPRESSORS
 fi
 
+if [ -n "$MONGODB_API_VERSION" ]; then
+    export MONGODB_API_VERSION=$MONGODB_API_VERSION
+fi
+
+
 export JAVA_HOME=/opt/java/jdk8
 
 if [ "$AUTH" != "noauth" ]; then

doc/api/pymongo/index.rst

@@ -54,6 +54,7 @@ Sub-modules:
    read_preferences
    results
    son_manipulator
+   server_api
    uri_parser
    write_concern
    event_loggers

doc/api/pymongo/server_api.rst

@@ -0,0 +1,11 @@
+:mod:`server_api` -- Support for MongoDB Versioned API
+======================================================
+
+.. automodule:: pymongo.server_api
+   :synopsis: Support for MongoDB Versioned API
+
+   .. autoclass:: pymongo.server_api.ServerApi
+      :members:
+
+   .. autoclass:: pymongo.server_api.ServerApiVersion
+      :members:

doc/changelog.rst

@@ -4,6 +4,19 @@ Changelog
 Changes in Version 3.12.0
 -------------------------
 
+Notable improvements
+....................
+
+- Support for MongoDB Versioned API, see :class:`~pymongo.server_api.ServerApi`.
+
+Issues Resolved
+...............
+
+See the `PyMongo 3.12 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 3.12 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=29594
+
 
 Changes in Version 3.11.1
 -------------------------

pymongo/client_options.py

@@ -125,6 +125,7 @@ def _parse_pool_options(options):
     event_listeners = options.get('event_listeners')
     appname = options.get('appname')
     driver = options.get('driver')
+    server_api = options.get('server_api')
     compression_settings = CompressionSettings(
         options.get('compressors', []),
         options.get('zlibcompressionlevel', -1))
@@ -138,7 +139,8 @@ def _parse_pool_options(options):
                        _EventListeners(event_listeners),
                        appname,
                        driver,
-                       compression_settings)
+                       compression_settings,
+                       server_api=server_api)
 
 
 class ClientOptions(object):

pymongo/client_session.py

@@ -297,6 +297,9 @@ def __init__(self, opts):
     def active(self):
         return self.state in (_TxnState.STARTING, _TxnState.IN_PROGRESS)
 
+    def starting(self):
+        return self.state == _TxnState.STARTING
+
     def reset(self):
         self.state = _TxnState.NONE
         self.sharded = False
@@ -762,6 +765,12 @@ def in_transaction(self):
         """
         return self._transaction.active()
 
+    @property
+    def _starting_transaction(self):
+        """True if this session is starting a multi-statement transaction.
+        """
+        return self._transaction.starting()
+
     @property
     def _pinned_address(self):
         """The mongos address this transaction was created on."""

pymongo/common.py

@@ -27,6 +27,7 @@
 from pymongo.compression_support import (validate_compressors,
                                          validate_zlib_compression_level)
 from pymongo.driver_info import DriverInfo
+from pymongo.server_api import ServerApi
 from pymongo.encryption_options import validate_auto_encryption_opts_or_none
 from pymongo.errors import ConfigurationError
 from pymongo.monitoring import _validate_event_listeners
@@ -525,6 +526,15 @@ def validate_driver_or_none(option, value):
     return value
 
 
+def validate_server_api_or_none(option, value):
+    """Validate the server_api keyword arg."""
+    if value is None:
+        return value
+    if not isinstance(value, ServerApi):
+        raise TypeError("%s must be an instance of ServerApi" % (option,))
+    return value
+
+
 def validate_is_callable_or_none(option, value):
     """Validates that 'value' is a callable."""
     if value is None:
@@ -640,6 +650,7 @@ def validate_tzinfo(dummy, value):
 NONSPEC_OPTIONS_VALIDATOR_MAP = {
     'connect': validate_boolean_or_string,
     'driver': validate_driver_or_none,
+    'server_api': validate_server_api_or_none,
     'fsync': validate_boolean_or_string,
     'minpoolsize': validate_non_negative_integer,
     'socketkeepalive': validate_boolean_or_string,

pymongo/database.py

@@ -704,6 +704,12 @@ def command(self, command, value=1, check=True,
         .. note:: :meth:`command` does **not** apply any custom TypeDecoders
            when decoding the command response.
 
+        .. note:: If this client has been configured to use MongoDB Versioned
+           API (see :ref:`versioned-api-ref`), then :meth:`command` will
+           automactically add API versioning options to the given command.
+           Explicitly adding API versioning options in the command and
+           declaring an API version on the client is not supported.
+
         .. versionchanged:: 3.6
            Added ``session`` parameter.
 

pymongo/message.py

@@ -307,6 +307,7 @@ def as_command(self, sock_info):
             self.name = 'explain'
             cmd = SON([('explain', cmd)])
         session = self.session
+        sock_info.add_server_api(cmd, session)
         if session:
             session._apply_to(cmd, False, self.read_preference)
             # Explain does not support readConcern.
@@ -892,6 +893,7 @@ def __init__(self, database_name, command, sock_info, operation_id,
         self.compress = True if sock_info.compression_context else False
         self.op_type = op_type
         self.codec = codec
+        sock_info.add_server_api(command, session)
 
     def _batch_command(self, docs):
         namespace = self.db_name + '.$cmd'

pymongo/mongo_client.py

@@ -498,8 +498,19 @@ def __init__(
             and automatically decrypt results. See
             :ref:`automatic-client-side-encryption` for an example.
 
+          | **Versioned API options:**
+          | (If not set explicitly, Versioned API will not be enabled.)
+
+          - `server_api`: A
+            :class:`~pymongo.server_api.ServerApi` which configures this
+            client to use Versioned API. See :ref:`versioned-api-ref` for
+            details.
+
         .. mongodoc:: connections
 
+        .. versionchanged:: 3.12
+           Added the ``server_api`` keyword argument.
+
         .. versionchanged:: 3.11
            Added the following keyword arguments and URI options:
 

pymongo/pool.py

@@ -58,6 +58,7 @@
 from pymongo.network import (command,
                              receive_message)
 from pymongo.read_preferences import ReadPreference
+from pymongo.server_api import _add_to_command
 from pymongo.server_type import SERVER_TYPE
 from pymongo.socket_checker import SocketChecker
 # Always use our backport so we always have support for IP address matching
@@ -294,7 +295,7 @@ class PoolOptions(object):
                  '__wait_queue_timeout', '__wait_queue_multiple',
                  '__ssl_context', '__ssl_match_hostname', '__socket_keepalive',
                  '__event_listeners', '__appname', '__driver', '__metadata',
-                 '__compression_settings')
+                 '__compression_settings', '__server_api')
 
     def __init__(self, max_pool_size=MAX_POOL_SIZE,
                  min_pool_size=MIN_POOL_SIZE,
@@ -303,8 +304,7 @@ def __init__(self, max_pool_size=MAX_POOL_SIZE,
                  wait_queue_multiple=None, ssl_context=None,
                  ssl_match_hostname=True, socket_keepalive=True,
                  event_listeners=None, appname=None, driver=None,
-                 compression_settings=None):
-
+                 compression_settings=None, server_api=None):
         self.__max_pool_size = max_pool_size
         self.__min_pool_size = min_pool_size
         self.__max_idle_time_seconds = max_idle_time_seconds
@@ -319,6 +319,7 @@ def __init__(self, max_pool_size=MAX_POOL_SIZE,
         self.__appname = appname
         self.__driver = driver
         self.__compression_settings = compression_settings
+        self.__server_api = server_api
         self.__metadata = copy.deepcopy(_METADATA)
         if appname:
             self.__metadata['application'] = {'name': appname}
@@ -462,6 +463,12 @@ def metadata(self):
         """
         return self.__metadata.copy()
 
+    @property
+    def server_api(self):
+        """A pymongo.server_api.ServerApi or None.
+        """
+        return self.__server_api
+
 
 def _negotiate_creds(all_credentials):
     """Return one credential that needs mechanism negotiation, if any.
@@ -672,6 +679,7 @@ def command(self, dbname, spec, slave_ok=False,
             raise ConfigurationError(
                 'Must be connected to MongoDB 3.4+ to use a collation.')
 
+        self.add_server_api(spec, session)
         if session:
             session._apply_to(spec, retryable_write, read_preference)
         self.send_cluster_time(spec, session, client)
@@ -861,6 +869,14 @@ def send_cluster_time(self, command, session, client):
         if self.max_wire_version >= 6 and client:
             client._send_cluster_time(command, session)
 
+    def add_server_api(self, command, session):
+        """Add server_api parameters."""
+        if (session and session.in_transaction and
+                not session._starting_transaction):
+            return
+        if self.opts.server_api:
+            _add_to_command(command, self.opts.server_api)
+
     def update_last_checkin_time(self):
         self.last_checkin_time = _time()