feat: add CMEK support (via synth)

Author: larkee

.gitignore

@@ -45,6 +45,7 @@ pip-log.txt
 
 # Built documentation
 docs/_build
+bigquery/docs/generated
 docs.metadata
 
 # Virtual environment

docs/conf.py

@@ -345,10 +345,11 @@
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
-    "python": ("http://python.readthedocs.org/en/latest/", None),
-    "google-auth": ("https://google-auth.readthedocs.io/en/stable", None),
+    "python": ("https://python.readthedocs.org/en/latest/", None),
+    "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None),
     "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,),
-    "grpc": ("https://grpc.io/grpc/python/", None),
+    "grpc": ("https://grpc.github.io/grpc/python/", None),
+    "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None),
 }
 
 

google/cloud/spanner_admin_database_v1/__init__.py

@@ -18,6 +18,7 @@
 from .services.database_admin import DatabaseAdminClient
 from .types.backup import Backup
 from .types.backup import BackupInfo
+from .types.backup import CreateBackupEncryptionConfig
 from .types.backup import CreateBackupMetadata
 from .types.backup import CreateBackupRequest
 from .types.backup import DeleteBackupRequest
@@ -27,6 +28,8 @@
 from .types.backup import ListBackupsRequest
 from .types.backup import ListBackupsResponse
 from .types.backup import UpdateBackupRequest
+from .types.common import EncryptionConfig
+from .types.common import EncryptionInfo
 from .types.common import OperationProgress
 from .types.spanner_database_admin import CreateDatabaseMetadata
 from .types.spanner_database_admin import CreateDatabaseRequest
@@ -40,6 +43,7 @@
 from .types.spanner_database_admin import ListDatabasesRequest
 from .types.spanner_database_admin import ListDatabasesResponse
 from .types.spanner_database_admin import OptimizeRestoredDatabaseMetadata
+from .types.spanner_database_admin import RestoreDatabaseEncryptionConfig
 from .types.spanner_database_admin import RestoreDatabaseMetadata
 from .types.spanner_database_admin import RestoreDatabaseRequest
 from .types.spanner_database_admin import RestoreInfo
@@ -51,13 +55,16 @@
 __all__ = (
     "Backup",
     "BackupInfo",
+    "CreateBackupEncryptionConfig",
     "CreateBackupMetadata",
     "CreateBackupRequest",
     "CreateDatabaseMetadata",
     "CreateDatabaseRequest",
     "Database",
     "DeleteBackupRequest",
     "DropDatabaseRequest",
+    "EncryptionConfig",
+    "EncryptionInfo",
     "GetBackupRequest",
     "GetDatabaseDdlRequest",
     "GetDatabaseDdlResponse",
@@ -72,6 +79,7 @@
     "ListDatabasesResponse",
     "OperationProgress",
     "OptimizeRestoredDatabaseMetadata",
+    "RestoreDatabaseEncryptionConfig",
     "RestoreDatabaseMetadata",
     "RestoreDatabaseRequest",
     "RestoreInfo",

google/cloud/spanner_admin_database_v1/proto/backup.proto

@@ -22,7 +22,6 @@ import "google/longrunning/operations.proto";
 import "google/protobuf/field_mask.proto";
 import "google/protobuf/timestamp.proto";
 import "google/spanner/admin/database/v1/common.proto";
-import "google/api/annotations.proto";
 
 option csharp_namespace = "Google.Cloud.Spanner.Admin.Database.V1";
 option go_package = "google.golang.org/genproto/googleapis/spanner/admin/database/v1;database";
@@ -109,6 +108,11 @@ message Backup {
   // restored database from the backup enters the `READY` state, the reference
   // to the backup is removed.
   repeated string referencing_databases = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The encryption information for the backup.
+  // If the encryption key protecting this resource is customer managed, then
+  // kms_key_version will be filled.
+  EncryptionInfo encryption_info = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // The request for [CreateBackup][google.spanner.admin.database.v1.DatabaseAdmin.CreateBackup].
@@ -133,6 +137,14 @@ message CreateBackupRequest {
 
   // Required. The backup to create.
   Backup backup = 3 [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. An encryption configuration describing the encryption type and key
+  // resources in Cloud KMS used to encrypt the backup. If no
+  // `encryption_config` is specified, the backup will use the same
+  // encryption configuration as the database by default, namely
+  // [encryption_type][google.spanner.admin.database.v1.CreateBackupEncryptionConfig.encryption_type] =
+  // USE_DATABASE_ENCRYPTION.
+  CreateBackupEncryptionConfig encryption_config = 4 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Metadata type for the operation returned by
@@ -384,3 +396,43 @@ message BackupInfo {
                                 type: "spanner.googleapis.com/Database"
                               }];
 }
+
+// Encryption configuration for the backup to create.
+message CreateBackupEncryptionConfig {
+  // Encryption types for the backup.
+  enum EncryptionType {
+    // Unspecified. Do not use.
+    ENCRYPTION_TYPE_UNSPECIFIED = 0;
+
+    // Use the same encryption configuration as the database. This is the
+    // default option when
+    // [encryption_config][CreateBackupEncryptionConfig.encryption_config] is
+    // empty. If the database is using customer managed encryption, the
+    // backup will be using the same KMS key.
+    USE_DATABASE_ENCRYPTION = 1;
+
+    // Enforce google default encryption.
+    GOOGLE_DEFAULT_ENCRYPTION = 2;
+
+    // Enforce customer managed encryption. If specified, the kms_key_name
+    // must provide a valid Cloud KMS key name.
+    CUSTOMER_MANAGED_ENCRYPTION = 3;
+  }
+
+  // Required. The encryption type of the backup.
+  EncryptionType encryption_type = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. The resource name of the Cloud KMS key that will be used to
+  // protect the backup. Once specified, the backup will enforce customer
+  // managed encryption, regardless of the database encryption type.
+  // This field should be set only when
+  // [encryption_type][google.spanner.admin.database.v1.CreateBackupEncryptionConfig.encryption_type] is
+  // CUSTOMER_MANAGED_ENCRYPTION. Values are of the form
+  // `projects/<project>/locations/<location>/keyRings/<key_ring>/cryptoKeys/<kms_key_name>`.
+  string kms_key_name = 2 [
+    (google.api.field_behavior) = OPTIONAL,
+    (google.api.resource_reference) = {
+      type: "cloudkms.googleapis.com/CryptoKey"
+    }
+  ];
+}

google/cloud/spanner_admin_database_v1/proto/common.proto

@@ -17,8 +17,9 @@ syntax = "proto3";
 package google.spanner.admin.database.v1;
 
 import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
 import "google/protobuf/timestamp.proto";
-import "google/api/annotations.proto";
+import "google/rpc/status.proto";
 
 option csharp_namespace = "Google.Cloud.Spanner.Admin.Database.V1";
 option go_package = "google.golang.org/genproto/googleapis/spanner/admin/database/v1;database";
@@ -27,6 +28,14 @@ option java_outer_classname = "CommonProto";
 option java_package = "com.google.spanner.admin.database.v1";
 option php_namespace = "Google\\Cloud\\Spanner\\Admin\\Database\\V1";
 option ruby_package = "Google::Cloud::Spanner::Admin::Database::V1";
+option (google.api.resource_definition) = {
+  type: "cloudkms.googleapis.com/CryptoKey"
+  pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}"
+};
+option (google.api.resource_definition) = {
+  type: "cloudkms.googleapis.com/CryptoKeyVersion"
+  pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}/cryptoKeyVersions/{crypto_key_version}"
+};
 
 // Encapsulates progress related information for a Cloud Spanner long
 // running operation.
@@ -42,3 +51,56 @@ message OperationProgress {
   // successfully.
   google.protobuf.Timestamp end_time = 3;
 }
+
+// Encryption configuration describing key resources in Cloud KMS used to
+// encrypt/decrypt a Cloud Spanner database.
+message EncryptionConfig {
+  // The resource name of the Cloud KMS key that was used to encrypt and decrypt
+  // the database. The form of the kms_key_name is
+  // `projects/<project>/locations/<location>/keyRings/<key_ring>/cryptoKeys\
+  // /<kms_key_name>`.
+  // api-linter: core::0122::name-suffix=disabled
+  // aip.dev/not-precedent: crypto key identifiers like this are listed as a
+  // canonical example of when field names would be ambiguous without the
+  // _name suffix and should therefore include it.
+  string kms_key_name = 2 [(google.api.resource_reference) = {
+                             type: "cloudkms.googleapis.com/CryptoKey"
+                           }];
+}
+
+// Encryption information for a given resource.
+// If this resource is protected with customer managed encryption, the in-use
+// Cloud KMS key versions will be specified along with their status.
+// CMEK is not currently available to end users.
+message EncryptionInfo {
+  // Possible encryption types for a resource.
+  enum Type {
+    // Encryption type was not specified, though data at rest remains encrypted.
+    TYPE_UNSPECIFIED = 0;
+
+    // The data backing this resource is encrypted at rest with a key that is
+    // fully managed by Google. No key version or status will be populated.
+    // This is the default state.
+    GOOGLE_DEFAULT_ENCRYPTION = 1;
+
+    // The data backing this resource is encrypted at rest with a key that is
+    // managed by the customer. The active version of the key. 'kms_key_version'
+    // will be populated, and 'encryption_status' may be populated.
+    CUSTOMER_MANAGED_ENCRYPTION = 2;
+  }
+
+  // Output only. The type of encryption used to protect this resource.
+  Type encryption_type = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. If present, the status of a recent encrypt/decrypt calls on underlying data
+  // for this resource. Regardless of status, data is always encrypted at rest.
+  google.rpc.Status encryption_status = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The Cloud KMS key versions used for a CMEK-protected Spanner resource.
+  string kms_key_version = 2 [
+    (google.api.field_behavior) = OUTPUT_ONLY,
+    (google.api.resource_reference) = {
+      type: "cloudkms.googleapis.com/CryptoKeyVersion"
+    }
+  ];
+}

google/cloud/spanner_admin_database_v1/proto/spanner_database_admin.proto

@@ -369,6 +369,12 @@ message Database {
   // about the restore source.
   RestoreInfo restore_info = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
 
+  // Output only. Custom encryption configuration (Cloud KMS keys).
+  // Applicable only for databases using the Customer Managed Encryption Keys
+  // feature.
+  EncryptionConfig encryption_config = 5
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+
   // Output only. The period in which Cloud Spanner retains all versions of data
   // for the database. This is the same as the value of version_retention_period
   // database option set using
@@ -436,6 +442,10 @@ message CreateDatabaseRequest {
   // statements execute atomically with the creation of the database:
   // if there is an error in any statement, the database is not created.
   repeated string extra_statements = 3 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional.
+  EncryptionConfig encryption_config = 4
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Metadata type for the operation returned by
@@ -673,6 +683,56 @@ message RestoreDatabaseRequest {
                          type: "spanner.googleapis.com/Backup"
                        }];
   }
+
+  // Optional. An encryption configuration describing the encryption type and key
+  // resources in Cloud KMS used to encrypt/decrypt the database to restore to.
+  // If no `encryption_config` is specified, the restored database will use
+  // the config default (if set) or the same encryption configuration as
+  // the backup by default, namely
+  // [encryption_type][google.spanner.admin.database.v1.RestoreDatabaseEncryptionConfig.encryption_type] =
+  // USE_CONFIG_DEFAULT_OR_DATABASE_ENCRYPTION.
+  RestoreDatabaseEncryptionConfig encryption_config = 4 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Encryption configuration for the database to restore to.
+message RestoreDatabaseEncryptionConfig {
+  // Encryption types for the database to be restored.
+  enum EncryptionType {
+    // Unspecified. Do not use.
+    ENCRYPTION_TYPE_UNSPECIFIED = 0;
+
+    // This is the default option when
+    // [encryption_config][RestoreDatabaseEncryptionConfig.encryption_config] is
+    // empty. It will first check whether there is a config default and use
+    // it if set. if not set, it will use the backup encryption setting. Note
+    // that the config default feature is a new feature that may not be
+    // available at the beginning.
+    USE_CONFIG_DEFAULT_OR_BACKUP_ENCRYPTION = 1;
+
+    // Enforce google default encryption.
+    GOOGLE_DEFAULT_ENCRYPTION = 2;
+
+    // Enforce customer managed encryption. If specified, the kms_key_name
+    // must provide a valid Cloud KMS key name.
+    CUSTOMER_MANAGED_ENCRYPTION = 3;
+  }
+
+  // Required. The encryption type of the restored database.
+  EncryptionType encryption_type = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. The resource name of the Cloud KMS key that will be used to
+  // encrypt/decrypt the database to restore to. Once specified, the database
+  // will enforce customer managed encryption, regardless of the backup
+  // encryption type. This field should be set only when
+  // [encryption_type][google.spanner.admin.database.v1.RestoreDatabaseEncryptionConfig.encryption_type] is
+  // CUSTOMER_MANAGED_ENCRYPTION. Values are of the form
+  // `projects/<project>/locations/<location>/keyRings/<key_ring>/cryptoKeys/<kms_key_name>`.
+  string kms_key_name = 2 [
+    (google.api.field_behavior) = OPTIONAL,
+    (google.api.resource_reference) = {
+      type: "cloudkms.googleapis.com/CryptoKey"
+    }
+  ];
 }
 
 // Metadata type for the long-running operation returned by

google/cloud/spanner_admin_database_v1/services/database_admin/async_client.py

@@ -33,6 +33,7 @@
 from google.cloud.spanner_admin_database_v1.services.database_admin import pagers
 from google.cloud.spanner_admin_database_v1.types import backup
 from google.cloud.spanner_admin_database_v1.types import backup as gsad_backup
+from google.cloud.spanner_admin_database_v1.types import common
 from google.cloud.spanner_admin_database_v1.types import spanner_database_admin
 from google.iam.v1 import iam_policy_pb2 as iam_policy  # type: ignore
 from google.iam.v1 import policy_pb2 as policy  # type: ignore
@@ -62,6 +63,12 @@ class DatabaseAdminAsyncClient:
 
     backup_path = staticmethod(DatabaseAdminClient.backup_path)
     parse_backup_path = staticmethod(DatabaseAdminClient.parse_backup_path)
+    crypto_key_path = staticmethod(DatabaseAdminClient.crypto_key_path)
+    parse_crypto_key_path = staticmethod(DatabaseAdminClient.parse_crypto_key_path)
+    crypto_key_version_path = staticmethod(DatabaseAdminClient.crypto_key_version_path)
+    parse_crypto_key_version_path = staticmethod(
+        DatabaseAdminClient.parse_crypto_key_version_path
+    )
     database_path = staticmethod(DatabaseAdminClient.database_path)
     parse_database_path = staticmethod(DatabaseAdminClient.parse_database_path)
     instance_path = staticmethod(DatabaseAdminClient.instance_path)

google/cloud/spanner_admin_database_v1/services/database_admin/client.py

@@ -37,6 +37,7 @@
 from google.cloud.spanner_admin_database_v1.services.database_admin import pagers
 from google.cloud.spanner_admin_database_v1.types import backup
 from google.cloud.spanner_admin_database_v1.types import backup as gsad_backup
+from google.cloud.spanner_admin_database_v1.types import common
 from google.cloud.spanner_admin_database_v1.types import spanner_database_admin
 from google.iam.v1 import iam_policy_pb2 as iam_policy  # type: ignore
 from google.iam.v1 import policy_pb2 as policy  # type: ignore
@@ -185,6 +186,53 @@ def parse_backup_path(path: str) -> Dict[str, str]:
         )
         return m.groupdict() if m else {}
 
+    @staticmethod
+    def crypto_key_path(
+        project: str, location: str, key_ring: str, crypto_key: str,
+    ) -> str:
+        """Return a fully-qualified crypto_key string."""
+        return "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}".format(
+            project=project,
+            location=location,
+            key_ring=key_ring,
+            crypto_key=crypto_key,
+        )
+
+    @staticmethod
+    def parse_crypto_key_path(path: str) -> Dict[str, str]:
+        """Parse a crypto_key path into its component segments."""
+        m = re.match(
+            r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/keyRings/(?P<key_ring>.+?)/cryptoKeys/(?P<crypto_key>.+?)$",
+            path,
+        )
+        return m.groupdict() if m else {}
+
+    @staticmethod
+    def crypto_key_version_path(
+        project: str,
+        location: str,
+        key_ring: str,
+        crypto_key: str,
+        crypto_key_version: str,
+    ) -> str:
+        """Return a fully-qualified crypto_key_version string."""
+        return "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}/cryptoKeyVersions/{crypto_key_version}".format(
+            project=project,
+            location=location,
+            key_ring=key_ring,
+            crypto_key=crypto_key,
+            crypto_key_version=crypto_key_version,
+        )
+
+    @staticmethod
+    def parse_crypto_key_version_path(path: str) -> Dict[str, str]:
+        """Parse a crypto_key_version path into its component segments."""
+        m = re.match(
+            r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/keyRings/(?P<key_ring>.+?)/cryptoKeys/(?P<crypto_key>.+?)/cryptoKeyVersions/(?P<crypto_key_version>.+?)$",
+            path,
+        )
+        return m.groupdict() if m else {}
+
     @staticmethod
     def database_path(project: str, instance: str, database: str,) -> str:
         """Return a fully-qualified database string."""