Qe equality follows (#3492)

* move return statement

* remove helpers from displayed code (#3414)

* capitalize initialisms in Go variable names

* customer-master-key instead of master-key

* update narrative to customer-master-key.txt and test highlighting

* remove emphasize-lines for now, no staging available on this branch

* remove duplicate tls code (#3427)

* fix admonition and add some highlighting

* update Go and Java driver versions

* additional emphasize lines in Quick Start

* aws fixes

* remove extra line

* azure updates

* gcp changes

* kmip changes

* kmip fixes

* always regen local key

* update dependencies

* add local CMK

* more fixes

* white space

* cc feedback

* update display for insert and find (#3444)

* switch direnv to dotenv in Node sample app (#3468)

* switch direnv to dotenv in Node sample app

* remove helper code from C# display

* Combine error message formatting with call to panic for Go tutorial

* improve auto encrypt sections (#3469)

* improve auto encrypt sections

* cc feedback

* Remove env variables (#3478)

* use appsettings

* add call to drop encrypted collection before creating (#3486)

* autobuilder

* Fix readmes and address comments

* kmip return line fix

---------

Co-authored-by: Mike Woofter <[email protected]>
Co-authored-by: Mike Woofter <[email protected]>
Co-authored-by: Chris Cho <[email protected]>

Author: 3 people

source/core/queryable-encryption/quick-start.txt

@@ -1,4 +1,4 @@
-.. note:: Generate a CMK from the Command Line
+.. tip:: Generate a CMK from the Command Line
 
    Use the following command to generate a {+cmk-abbr+}
    from a Unix shell or PowerShell:

source/core/queryable-encryption/tutorials/aws/aws-automatic.txt

@@ -1,10 +1,9 @@
-namespace QueryableEncryption
+namespace QueryableEncryption;
+
+public class Program
 {
-    public class Program
+    public static void Main(string[] args)
     {
-        public static void Main(string[] args)
-        {
-            QueryableEncryptionTutorial.RunExample();
-        }
+        QueryableEncryptionTutorial.RunExample();
     }
 }

source/core/queryable-encryption/tutorials/azure/azure-automatic.txt

@@ -8,7 +8,15 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
     <PackageReference Include="MongoDB.Driver" Version="2.20.0" />
   </ItemGroup>
+
+  <ItemGroup>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
 
 </Project>

source/core/queryable-encryption/tutorials/gcp/gcp-automatic.txt

@@ -1,14 +1,21 @@
 using System.Security.Cryptography;
 using System.Security.Cryptography.X509Certificates;
+using Microsoft.Extensions.Configuration;
 using MongoDB.Bson;
 using MongoDB.Driver;
 using MongoDB.Driver.Encryption;
 
 namespace QueryableEncryption;
 
-public static class QueryableEncryptionHelpers
+public class QueryableEncryptionHelpers
 {
-    public static Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProviderCredentials(string kmsProvider,
+    private readonly IConfigurationRoot _appSettings;
+    public QueryableEncryptionHelpers(IConfigurationRoot appSettings)
+    {
+        _appSettings = appSettings;
+    }
+    
+    public Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProviderCredentials(string kmsProvider,
         bool generateNewLocalKey)
     {
         if (kmsProvider == "aws")
@@ -17,8 +24,8 @@ public static Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProv
             var kmsProviderCredentials = new Dictionary<string, IReadOnlyDictionary<string, object>>();
             var awsKmsOptions = new Dictionary<string, object>
             {
-                { "accessKeyId", Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID")! }, // Your AWS access key ID
-                { "secretAccessKey", Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY")! } // Your AWS secret access key
+                { "accessKeyId", _appSettings["Aws:AccessKeyId"] }, // Your AWS access key ID
+                { "secretAccessKey", _appSettings["Aws:SecretAccessKey"] } // Your AWS secret access key
             };
             kmsProviderCredentials.Add(kmsProvider, awsKmsOptions);
             // end-aws-kms-credentials
@@ -30,9 +37,9 @@ public static Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProv
             var kmsProviderCredentials = new Dictionary<string, IReadOnlyDictionary<string, object>>();
             var azureKmsOptions = new Dictionary<string, object>
             {
-                { "tenantId", Environment.GetEnvironmentVariable("AZURE_TENANT_ID")! }, // Your Azure tenant ID
-                { "clientId", Environment.GetEnvironmentVariable("AZURE_CLIENT_ID")! }, // Your Azure client ID
-                { "clientSecret", Environment.GetEnvironmentVariable("AZURE_CLIENT_SECRET")! } // Your Azure client secret
+                { "tenantId", _appSettings["Azure:TenantId"] }, // Your Azure tenant ID
+                { "clientId", _appSettings["Azure:ClientId"] }, // Your Azure client ID
+                { "clientSecret", _appSettings["Azure:ClientSecret"] } // Your Azure client secret
             };
             kmsProviderCredentials.Add(kmsProvider, azureKmsOptions);
             // end-azure-kms-credentials
@@ -44,8 +51,8 @@ public static Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProv
             var kmsProviderCredentials = new Dictionary<string, IReadOnlyDictionary<string, object>>();
             var gcpKmsOptions = new Dictionary<string, object>
             {
-                { "email", Environment.GetEnvironmentVariable("GCP_EMAIL")! }, // Your GCP email
-                { "privateKey", Environment.GetEnvironmentVariable("GCP_PRIVATE_KEY")! } // Your GCP private key
+                { "email", _appSettings["Gcp:Email"] }, // Your GCP email
+                { "privateKey", _appSettings["Gcp:PrivateKey"] } // Your GCP private key
             };
             kmsProviderCredentials.Add(kmsProvider, gcpKmsOptions);
             // end-gcp-kms-credentials
@@ -57,7 +64,7 @@ public static Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProv
             var kmsProviderCredentials = new Dictionary<string, IReadOnlyDictionary<string, object>>();
             var kmipKmsOptions = new Dictionary<string, object>
             {
-                { "endpoint", Environment.GetEnvironmentVariable("KMIP_KMS_ENDPOINT")! } // Your KMIP KMS endpoint
+                { "endpoint", _appSettings["Kmip:KmsEndpoint"] } // Your KMIP KMS endpoint
             };
             kmsProviderCredentials.Add(kmsProvider, kmipKmsOptions);
             // end-kmip-kms-credentials
@@ -67,7 +74,7 @@ public static Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProv
         {
             if (generateNewLocalKey)
             {
-                File.Delete("master-key.txt");
+                File.Delete("customer-master-key.txt");
 
                 // start-generate-local-key
                 using var randomNumberGenerator = RandomNumberGenerator.Create();
@@ -76,7 +83,7 @@ public static Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProv
                     var bytes = new byte[96];
                     randomNumberGenerator.GetBytes(bytes);
                     var localCustomerMasterKeyBase64Write = Convert.ToBase64String(bytes);
-                    File.WriteAllText("master-key.txt", localCustomerMasterKeyBase64Write);
+                    File.WriteAllText("customer-master-key.txt", localCustomerMasterKeyBase64Write);
                 }
                 catch (Exception e)
                 {
@@ -90,7 +97,7 @@ public static Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProv
             var kmsProviderCredentials = new Dictionary<string, IReadOnlyDictionary<string, object>>();
             try
             {
-                var localCustomerMasterKeyBase64Read = File.ReadAllText("master-key.txt");
+                var localCustomerMasterKeyBase64Read = File.ReadAllText("customer-master-key.txt");
                 var localCustomerMasterKeyBytes = Convert.FromBase64String(localCustomerMasterKeyBase64Read);
 
                 var localOptions = new Dictionary<string, object>
@@ -100,27 +107,27 @@ public static Dictionary<string, IReadOnlyDictionary<string, object>> GetKmsProv
 
                 kmsProviderCredentials.Add(kmsProvider, localOptions);
             }
+            // end-get-local-key
             catch (Exception e)
             {
                 Console.WriteLine(e.Message);
             }
-            // end-get-local-key
             return kmsProviderCredentials;
 
         }
 
         throw new Exception("Invalid KMS provider string");
     }
 
-    public static BsonDocument GetCustomerMasterKeyCredentials(string kmsProvider)
+    public BsonDocument GetCustomerMasterKeyCredentials(string kmsProvider)
     {
         if (kmsProvider == "aws")
         {
             // start-aws-cmk-credentials
             var customerMasterKeyCredentials = new BsonDocument
             {
-                { "key", Environment.GetEnvironmentVariable("AWS_KEY_ARN")}, // Your AWS Key ARN
-                { "region", Environment.GetEnvironmentVariable("AWS_KEY_REGION") } // Your AWS Key Region
+                { "key", _appSettings["Aws:KeyArn"] }, // Your AWS Key ARN
+                { "region", _appSettings["Aws:KeyRegion"] } // Your AWS Key Region
             };
             // end-aws-cmk-credentials
             return customerMasterKeyCredentials;
@@ -130,8 +137,8 @@ public static BsonDocument GetCustomerMasterKeyCredentials(string kmsProvider)
             // start-azure-cmk-credentials
             var customerMasterKeyCredentials = new BsonDocument
             {
-                { "keyVaultEndpoint", Environment.GetEnvironmentVariable("AZURE_KEY_VAULT_ENDPOINT") }, // Your Azure Key Vault Endpoint
-                { "keyName", Environment.GetEnvironmentVariable("AZURE_KEY_NAME") } // Your Azure Key Name
+                { "keyVaultEndpoint", _appSettings["Azure:KeyVaultEndpoint"] }, // Your Azure Key Vault Endpoint
+                { "keyName", _appSettings["Azure:KeyName"] } // Your Azure Key Name
             };
             // end-azure-cmk-credentials
             return customerMasterKeyCredentials;
@@ -141,43 +148,44 @@ public static BsonDocument GetCustomerMasterKeyCredentials(string kmsProvider)
             // start-gcp-cmk-credentials
             var customerMasterKeyCredentials = new BsonDocument
             {
-                { "projectId", Environment.GetEnvironmentVariable("GCP_PROJECT_ID") }, // Your GCP Project ID
-                { "location", Environment.GetEnvironmentVariable("GCP_LOCATION") }, // Your GCP Key Location
-                { "keyRing", Environment.GetEnvironmentVariable("GCP_KEY_RING") }, // Your GCP Key Ring
-                { "keyName", Environment.GetEnvironmentVariable("GCP_KEY_NAME")} // Your GCP Key Name
+                { "projectId", _appSettings["Gcp:ProjectId"] }, // Your GCP Project ID
+                { "location", _appSettings["Gcp:Location"] }, // Your GCP Key Location
+                { "keyRing", _appSettings["Gcp:KeyRing"] }, // Your GCP Key Ring
+                { "keyName", _appSettings["Gcp:KeyName"] } // Your GCP Key Name
             };
             // end-gcp-cmk-credentials
-            return customerMasterKeyCredentials;           
+            return customerMasterKeyCredentials;
         }
-        else if (kmsProvider == "kmip")
+        else if (kmsProvider == "kmip" || kmsProvider == "local")
         {
             // start-kmip-local-cmk-credentials
             var customerMasterKeyCredentials = new BsonDocument();
             // end-kmip-local-cmk-credentials
             return customerMasterKeyCredentials;
-        } else
-        { 
+        }
+        else
+        {
             throw new Exception("Invalid KMS provider string");
         }
     }
 
-    public static AutoEncryptionOptions GetAutoEncryptionOptions(CollectionNamespace keyVaultNamespace,
+    public AutoEncryptionOptions GetAutoEncryptionOptions(CollectionNamespace keyVaultNamespace,
         IReadOnlyDictionary<string, IReadOnlyDictionary<string, object>> kmsProviderCredentials)
     {
         var kmsProvider = kmsProviderCredentials.Keys.First();
 
         if (kmsProvider == "kmip")
         {
+            var tlsOptions = GetKmipTlsOptions();
+
             // start-kmip-encryption-options
             var extraOptions = new Dictionary<string, object>
             {
-                { "cryptSharedLibPath", Environment.GetEnvironmentVariable(
-                    "CRYPT_SHARED_LIB_PATH") } // Path to your Automatic Encryption Shared Library
+                { "cryptSharedLibPath", _appSettings["CryptSharedLibPath"] } // Path to your Automatic Encryption Shared Library
             };
 
-            var tlsOptions = GetKmipTlsOptions();
-
-            var autoEncryptionOptions = new AutoEncryptionOptions(keyVaultNamespace,
+            var autoEncryptionOptions = new AutoEncryptionOptions(
+                keyVaultNamespace,
                 kmsProviderCredentials,
                 extraOptions: extraOptions,
                 tlsOptions: tlsOptions);
@@ -189,11 +197,11 @@ public static AutoEncryptionOptions GetAutoEncryptionOptions(CollectionNamespace
             // start-auto-encryption-options
             var extraOptions = new Dictionary<string, object>
             {
-                { "cryptSharedLibPath", Environment.GetEnvironmentVariable(
-                    "CRYPT_SHARED_LIB_PATH") } // Path to your Automatic Encryption Shared Library
+                { "cryptSharedLibPath", _appSettings["CryptSharedLibPath"] } // Path to your Automatic Encryption Shared Library
             };
 
-            var autoEncryptionOptions = new AutoEncryptionOptions(keyVaultNamespace,
+            var autoEncryptionOptions = new AutoEncryptionOptions(
+                keyVaultNamespace,
                 kmsProviderCredentials,
                 extraOptions: extraOptions);
             // end-auto-encryption-options
@@ -202,7 +210,7 @@ public static AutoEncryptionOptions GetAutoEncryptionOptions(CollectionNamespace
         }
     }
 
-    public static ClientEncryption GetClientEncryption(IMongoClient keyVaultClient,
+    public ClientEncryption GetClientEncryption(IMongoClient keyVaultClient,
         CollectionNamespace keyVaultNamespace, Dictionary<string, IReadOnlyDictionary<string, object>> kmsProviderCredentials)
     {
         var kmsProvider = kmsProviderCredentials.Keys.First();
@@ -236,12 +244,12 @@ public static ClientEncryption GetClientEncryption(IMongoClient keyVaultClient,
         }
     }
 
-    private static Dictionary<string, SslSettings> GetKmipTlsOptions()
+    private Dictionary<string, SslSettings> GetKmipTlsOptions()
     {
         // start-tls-options
         var tlsOptions = new Dictionary<string, SslSettings>();
         var sslSettings = new SslSettings();
-        var clientCertificate = new X509Certificate2(Environment.GetEnvironmentVariable("KMIP_TLS_CERT_P12")!); // Full path to your client certificate p12 file
+        var clientCertificate = new X509Certificate2(_appSettings["Kmip:TlsCertP12"]!); // Full path to your client certificate p12 file
         sslSettings.ClientCertificates = new[] { clientCertificate };
         tlsOptions.Add("kmip", sslSettings);
         // end-tls-options

source/core/queryable-encryption/tutorials/kmip/kmip-automatic.txt

@@ -1,7 +1,7 @@
+using Microsoft.Extensions.Configuration;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Conventions;
 using MongoDB.Driver;
-using static QueryableEncryption.QueryableEncryptionHelpers;
 
 namespace QueryableEncryption;
 
@@ -11,7 +11,7 @@ public static async void RunExample()
     {
         var camelCaseConvention = new ConventionPack { new CamelCaseElementNameConvention() };
         ConventionRegistry.Register("CamelCase", camelCaseConvention, type => true);
-        
+
         // start-setup-application-variables
         // KMS provider name should be one of the following: "aws", "gcp", "azure", "kmip" or "local"
         const string kmsProviderName = "<your KMS provider name>";
@@ -21,20 +21,23 @@ public static async void RunExample()
             CollectionNamespace.FromFullName($"{keyVaultDatabaseName}.{keyVaultCollectionName}");
         const string encryptedDatabaseName = "medicalRecords";
         const string encryptedCollectionName = "patients";
-        var uri = Environment.GetEnvironmentVariable("MONGODB_URI"); // Your connection URI 
+
+        var appSettings = new ConfigurationBuilder().AddJsonFile("appsettings.json").Build();
+        var uri = appSettings["MongoDbUri"];
         // end-setup-application-variables
-        
-        var kmsProviderCredentials = GetKmsProviderCredentials(kmsProviderName,
-            generateNewLocalKey: false);
-        
+
+        var qeHelpers = new QueryableEncryptionHelpers(appSettings);
+        var kmsProviderCredentials = qeHelpers.GetKmsProviderCredentials(kmsProviderName,
+            generateNewLocalKey: true);
+
         // start-create-client
         var clientSettings = MongoClientSettings.FromConnectionString(uri);
-        clientSettings.AutoEncryptionOptions = GetAutoEncryptionOptions(
+        clientSettings.AutoEncryptionOptions = qeHelpers.GetAutoEncryptionOptions(
             keyVaultNamespace,
             kmsProviderCredentials);
         var encryptedClient = new MongoClient(clientSettings);
         // end-create-client
-        
+
         var keyDatabase = encryptedClient.GetDatabase(keyVaultDatabaseName);
 
         // Drop the collection in case you created it in a previous run of this application.
@@ -51,7 +54,7 @@ public static async void RunExample()
                         { "keyId", BsonNull.Value },
                         { "path", "record.ssn" },
                         { "bsonType", "string" },
-                        { "queries", new BsonDocument("queryType", "equality") } 
+                        { "queries", new BsonDocument("queryType", "equality") }
                     },
                     new BsonDocument
                     {
@@ -63,25 +66,29 @@ public static async void RunExample()
             }
         };
         // end-encrypted-fields-map
-        
+
         var patientDatabase = encryptedClient.GetDatabase(encryptedDatabaseName);
-        var clientEncryption = GetClientEncryption(encryptedClient,
+        patientDatabase.DropCollection(encryptedCollectionName);
+
+        var clientEncryption = qeHelpers.GetClientEncryption(encryptedClient,
             keyVaultNamespace,
             kmsProviderCredentials);
-        
+
+        var customerMasterKeyCredentials = qeHelpers.GetCustomerMasterKeyCredentials(kmsProviderName);
+
         // start-create-encrypted-collection
         var createCollectionOptions = new CreateCollectionOptions<Patient>
         {
-            EncryptedFields = encryptedFields 
+            EncryptedFields = encryptedFields
         };
-        
-        clientEncryption.CreateEncryptedCollection(patientDatabase, 
+
+        clientEncryption.CreateEncryptedCollection(patientDatabase,
             encryptedCollectionName,
             createCollectionOptions,
             kmsProviderName,
-            GetCustomerMasterKeyCredentials(kmsProviderName));
+            customerMasterKeyCredentials);
         // end-create-encrypted-collection
-        
+
         // start-insert-document
         var patient = new Patient
         {
@@ -100,10 +107,10 @@ public static async void RunExample()
 
         var encryptedCollection = encryptedClient.GetDatabase(encryptedDatabaseName).
             GetCollection<Patient>(encryptedCollectionName);
-        
+
         encryptedCollection.InsertOne(patient);
         // end-insert-document
-    
+
         // start-find-document
         var ssnFilter = Builders<Patient>.Filter.Eq("record.ssn", patient.Record.Ssn);
         var findResult = await encryptedCollection.Find(ssnFilter).ToCursorAsync();