RUBY-3226 Update libmongocrypt payloads to new QE protocol (#2718)

* update the QE specs

* bump libmongocrypt-helper to 1.8.0

* update prose specs (server version >= 7, linting)

* linting

* didn't mean to commit this file

* more prose specs to be updated

* use the right version of fle2v2 specs

they do a funky thing where they treat nil values as implying the absence of a field;
had to do some work arounds to make that work. Hopefully it doesn't break anything
new...

* disallow encrypted collections on older servers

* eccCollection is not to be created

* enable linting on a couple more files

* allow constants to be loaded after the file loads

* some of these constants need to be in the utils namespace

* don't need both the min_fcv and min_server_version constraints

Author: jamis

.rubocop.yml

@@ -44,9 +44,56 @@ Bundler/OrderedGems:
 Gemspec/OrderedDependencies:
   Enabled: false
 
+Layout/SpaceInsideArrayLiteralBrackets:
+  EnforcedStyle: space
+
+Layout/SpaceInsidePercentLiteralDelimiters:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 20
+
+RSpec/BeforeAfterAll:
+  Enabled: false
+
+# Ideally, we'd use this one, too, but our tests have not historically followed
+# this style and it's not worth changing right now, IMO
+RSpec/DescribeClass:
+  Enabled: false
+
+RSpec/ImplicitExpect:
+  EnforcedStyle: is_expected
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
+RSpec/NestedGroups:
+  Enabled: false
+
 Style/Documentation:
   Exclude:
     - 'spec/**/*'
 
-Metrics/MethodLength:
-  Max: 20
+Style/ModuleFunction:
+  EnforcedStyle: extend_self
+
+Style/OptionalBooleanParameter:
+  Enabled: false
+
+Style/ParallelAssignment:
+  Enabled: false
+
+Style/TernaryParentheses:
+  EnforcedStyle: require_parentheses_when_complex
+
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+
+Style/TrailingCommaInHashLiteral:
+  Enabled: false

gemfiles/standard.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
-# rubocop:todo all
 
+# rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength
 def standard_dependencies
   gem 'yard'
   gem 'ffi'
@@ -29,7 +29,7 @@ def standard_dependencies
 
     # for static analysis -- ignore ruby < 2.6 because of rubocop
     # version incompatibilities
-    if RUBY_VERSION > "2.5.99"
+    if RUBY_VERSION > '2.5.99'
       gem 'rubocop', '~> 1.45.1'
       gem 'rubocop-performance', '~> 1.16.0'
       gem 'rubocop-rake', '~> 0.6.0'
@@ -66,7 +66,6 @@ def standard_dependencies
     gem 'solargraph', platforms: :mri
   end
 
-  if ENV['FLE'] == 'helper'
-    gem 'libmongocrypt-helper', '~> 1.7.0'
-  end
+  gem 'libmongocrypt-helper', '~> 1.8.0' if ENV['FLE'] == 'helper'
 end
+# rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength

lib/mongo/collection/queryable_encryption.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:todo all
 
 # Copyright (C) 2014-2022 MongoDB Inc.
 #
@@ -22,6 +21,8 @@ class Collection
     #
     # @api private
     module QueryableEncryption
+      # The minimum wire version for QE2 support
+      QE2_MIN_WIRE_VERSION = 21
 
       # Creates auxiliary collections and indices for queryable encryption if necessary.
       #
@@ -32,32 +33,17 @@ module QueryableEncryption
       #
       # @return [ Result ] The result of provided block.
       def maybe_create_qe_collections(encrypted_fields, client, session)
-        encrypted_fields = if encrypted_fields
-          encrypted_fields
-        elsif encrypted_fields_map
-          encrypted_fields_map[namespace] || {}
-        else
-          {}
-        end
-        if encrypted_fields.empty?
-          return yield
-        end
+        encrypted_fields = encrypted_fields_from(encrypted_fields)
+        return yield if encrypted_fields.empty?
+
+        check_wire_version!
 
         emm_collections(encrypted_fields).each do |coll|
           context = Operation::Context.new(client: client, session: session)
-          Operation::Create.new(
-            selector: {
-              create: coll,
-              clusteredIndex: {
-                key: {
-                  _id: 1
-                },
-                unique: true
-              }
-            },
-            db_name: database.name,
-          ).execute(next_primary(nil, session), context: context)
+          create_operation_for(coll)
+            .execute(next_primary(nil, session), context: context)
         end
+
         yield(encrypted_fields).tap do |result|
           indexes.create_one(__safeContent__: 1) if result
         end
@@ -73,31 +59,25 @@ def maybe_create_qe_collections(encrypted_fields, client, session)
       # @return [ Result ] The result of provided block.
       def maybe_drop_emm_collections(encrypted_fields, client, session)
         encrypted_fields = if encrypted_fields
-          encrypted_fields
-        elsif encrypted_fields_map
-          if encrypted_fields_map[namespace]
-            encrypted_fields_map[namespace]
-          else
-            database.list_collections(filter: { name: name })
-              .first
-              &.fetch(:options, {})
-              &.fetch(:encryptedFields, {}) || {}
-          end
-        else
-          {}
-        end
-        if encrypted_fields.empty?
-          return yield
-        end
+                             encrypted_fields
+                           elsif encrypted_fields_map
+                             encrypted_fields_for_drop_from_map
+                           else
+                             {}
+                           end
+
+        return yield if encrypted_fields.empty?
+
         emm_collections(encrypted_fields).each do |coll|
           context = Operation::Context.new(client: client, session: session)
           operation = Operation::Drop.new(
             selector: { drop: coll },
             db_name: database.name,
-            session: session,
+            session: session
           )
           do_drop(operation, session, context)
         end
+
         yield
       end
 
@@ -112,11 +92,66 @@ def maybe_drop_emm_collections(encrypted_fields, client, session)
       def emm_collections(encrypted_fields)
         [
           encrypted_fields['escCollection'] || "enxcol_.#{name}.esc",
-          encrypted_fields['eccCollection'] || "enxcol_.#{name}.ecc",
           encrypted_fields['ecocCollection'] || "enxcol_.#{name}.ecoc",
         ]
       end
 
+      # Creating encrypted collections is only supported on 7.0.0 and later
+      # (wire version 21+).
+      #
+      # @raise [ Mongo::Error ] if the wire version is not
+      #   recent enough
+      def check_wire_version!
+        return unless next_primary.max_wire_version < QE2_MIN_WIRE_VERSION
+
+        raise Mongo::Error,
+              'Driver support of Queryable Encryption is incompatible with server. ' \
+              'Upgrade server to use Queryable Encryption.'
+      end
+
+      # Tries to return the encrypted fields from the argument. If the argument
+      # is nil, tries to find the encrypted fields from the
+      # encrypted_fields_map.
+      #
+      # @param [ Hash | nil ] fields the encrypted fields
+      #
+      # @return [ Hash ] the encrypted fields
+      def encrypted_fields_from(fields)
+        fields ||
+          (encrypted_fields_map && encrypted_fields_map[namespace]) ||
+          {}
+      end
+
+      # Tries to return the encrypted fields from the {{encrypted_fields_map}}
+      # value, for the current namespace.
+      #
+      # @return [ Hash | nil ] the encrypted fields, if found
+      def encrypted_fields_for_drop_from_map
+        encrypted_fields_map[namespace] ||
+          database.list_collections(filter: { name: name })
+                  .first
+                  &.fetch(:options, {})
+                  &.fetch(:encryptedFields, {}) ||
+          {}
+      end
+
+      # Returns a new create operation for the given collection.
+      #
+      # @param [ String ] coll the name of the collection to create.
+      #
+      # @return [ Operation::Create ] the new create operation.
+      def create_operation_for(coll)
+        Operation::Create.new(
+          selector: {
+            create: coll,
+            clusteredIndex: {
+              key: { _id: 1 },
+              unique: true
+            }
+          },
+          db_name: database.name
+        )
+      end
     end
   end
 end

lib/mongo/crypt/kms/azure/credentials_retriever.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# rubocop:todo all
 
 # Copyright (C) 2019-2021 MongoDB Inc.
 #