[storage] testing: use multiple projects (#4048)

* [storage] testing: use multiple projects

We still need to use the old project for some tests.

fixes #4033
fixes #4029

* remove print

* use uuid instead of time.time()

* lint fix

Author: Takashi Matsuo

storage/cloud-client/acl_test.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import uuid
 
 from google.cloud import storage
@@ -39,13 +40,20 @@
 @pytest.fixture(scope="module")
 def test_bucket():
     """Yields a bucket that is deleted after the test completes."""
+
+    # The new projects have uniform bucket-level access and our tests don't
+    # pass with those buckets. We need to use the old main project for now.
+    original_value = os.environ['GOOGLE_CLOUD_PROJECT']
+    os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT']
     bucket = None
     while bucket is None or bucket.exists():
         bucket_name = "acl-test-{}".format(uuid.uuid4())
         bucket = storage.Client().bucket(bucket_name)
     bucket.create()
     yield bucket
     bucket.delete(force=True)
+    # Set the value back.
+    os.environ['GOOGLE_CLOUD_PROJECT'] = original_value
 
 
 @pytest.fixture

storage/cloud-client/hmac_samples_test.py

@@ -31,8 +31,14 @@
 import storage_get_hmac_key
 import storage_list_hmac_keys
 
-
-PROJECT_ID = os.environ["GOOGLE_CLOUD_PROJECT"]
+# We are reaching maximum number of HMAC keys on the service account.
+# We change the service account based on the value of
+# RUN_TESTS_SESSION in noxfile_config.py.
+# The reason we can not use multiple project is that our new projects
+# are enforced to have
+# 'constraints/iam.disableServiceAccountKeyCreation' policy.
+
+PROJECT_ID = os.environ["MAIN_GOOGLE_CLOUD_PROJECT"]
 SERVICE_ACCOUNT_EMAIL = os.environ["HMAC_KEY_TEST_SERVICE_ACCOUNT"]
 STORAGE_CLIENT = storage.Client(project=PROJECT_ID)
 

storage/cloud-client/iam_test.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import re
 import time
 import uuid
@@ -36,7 +37,7 @@
 )
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def bucket():
     bucket = None
     while bucket is None or bucket.exists():
@@ -50,6 +51,26 @@ def bucket():
     bucket.delete(force=True)
 
 
+@pytest.fixture(scope="function")
+def public_bucket():
+    # The new projects don't allow to make a bucket available to public, so
+    # we need to use the old main project for now.
+    original_value = os.environ['GOOGLE_CLOUD_PROJECT']
+    os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT']
+    bucket = None
+    while bucket is None or bucket.exists():
+        storage_client = storage.Client()
+        bucket_name = "test-iam-{}".format(uuid.uuid4())
+        bucket = storage_client.bucket(bucket_name)
+        bucket.iam_configuration.uniform_bucket_level_access_enabled = True
+    storage_client.create_bucket(bucket)
+    yield bucket
+    time.sleep(3)
+    bucket.delete(force=True)
+    # Set the value back.
+    os.environ['GOOGLE_CLOUD_PROJECT'] = original_value
+
+
 def test_view_bucket_iam_members(capsys, bucket):
     storage_view_bucket_iam_members.view_bucket_iam_members(bucket.name)
     assert re.match("Role: .*, Members: .*", capsys.readouterr().out)
@@ -87,10 +108,11 @@ def test_add_bucket_conditional_iam_binding(bucket):
     )
 
 
-def test_remove_bucket_iam_member(bucket):
-    storage_remove_bucket_iam_member.remove_bucket_iam_member(bucket.name, ROLE, MEMBER)
+def test_remove_bucket_iam_member(public_bucket):
+    storage_remove_bucket_iam_member.remove_bucket_iam_member(
+        public_bucket.name, ROLE, MEMBER)
 
-    policy = bucket.get_iam_policy(requested_policy_version=3)
+    policy = public_bucket.get_iam_policy(requested_policy_version=3)
     assert not any(
         binding["role"] == ROLE and MEMBER in binding["members"]
         for binding in policy.bindings
@@ -114,13 +136,13 @@ def test_remove_bucket_conditional_iam_binding(bucket):
     )
 
 
-def test_set_bucket_public_iam(bucket):
+def test_set_bucket_public_iam(public_bucket):
     role = "roles/storage.objectViewer"
     member = "allUsers"
     storage_set_bucket_public_iam.set_bucket_public_iam(
-        bucket.name, role, member
+        public_bucket.name, role, member
     )
-    policy = bucket.get_iam_policy(requested_policy_version=3)
+    policy = public_bucket.get_iam_policy(requested_policy_version=3)
     assert any(
         binding["role"] == role and member in binding["members"]
         for binding in policy.bindings

storage/cloud-client/noxfile_config.py

@@ -0,0 +1,84 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default TEST_CONFIG_OVERRIDE for python repos.
+
+# You can copy this file into your directory, then it will be inported from
+# the noxfile.py.
+
+# The source of truth:
+# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py
+
+import os
+
+
+# We are reaching maximum number of HMAC keys on the service account.
+# We change the service account based on the value of
+# RUN_TESTS_SESSION. The reason we can not use multiple project is
+# that our new projects are enforced to have
+# 'constraints/iam.disableServiceAccountKeyCreation' policy.
+def get_service_account_email():
+    session = os.environ.get('RUN_TESTS_SESSION')
+    if session == 'py-3.6':
+        return ('py36-storage-test@'
+                'python-docs-samples-tests.iam.gserviceaccount.com')
+    if session == 'py-3.7':
+        return ('py37-storage-test@'
+                'python-docs-samples-tests.iam.gserviceaccount.com')
+    if session == 'py-3.8':
+        return ('py38-storage-test@'
+                'python-docs-samples-tests.iam.gserviceaccount.com')
+    return os.environ['HMAC_KEY_TEST_SERVICE_ACCOUNT']
+
+
+# We change the value of CLOUD_KMS_KEY based on the value of
+# RUN_TESTS_SESSION.
+def get_cloud_kms_key():
+    session = os.environ.get('RUN_TESTS_SESSION')
+    if session == 'py-3.6':
+        return ('projects/python-docs-samples-tests-py36/locations/us/'
+                'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key')
+    if session == 'py-3.7':
+        return ('projects/python-docs-samples-tests-py37/locations/us/'
+                'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key')
+    if session == 'py-3.8':
+        return ('projects/python-docs-samples-tests-py38/locations/us/'
+                'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key')
+    return os.environ['CLOUD_KMS_KEY']
+
+
+TEST_CONFIG_OVERRIDE = {
+    # You can opt out from the test for specific Python versions.
+    'ignored_versions': ["2.7"],
+
+    # An envvar key for determining the project id to use. Change it
+    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
+    # build specific Cloud project. You can also use your own string
+    # to use your own Cloud project.
+    # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT',
+    'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
+
+    # A dictionary you want to inject into your test. Don't put any
+    # secrets here. These values will override predefined values.
+    'envs': {
+        'HMAC_KEY_TEST_SERVICE_ACCOUNT': get_service_account_email(),
+        'CLOUD_KMS_KEY': get_cloud_kms_key(),
+        # Some tests can not use multiple projects because of several reasons:
+        # 1. The new projects is enforced to have the
+        # 'constraints/iam.disableServiceAccountKeyCreation' policy.
+        # 2. The new projects buckets need to have universal permission model.
+        # For those tests, we'll use the original project.
+        'MAIN_GOOGLE_CLOUD_PROJECT': 'python-docs-samples-tests'
+    },
+}

storage/cloud-client/snippets_test.py

@@ -105,6 +105,24 @@ def test_bucket():
     bucket.delete(force=True)
 
 
+@pytest.fixture(scope="function")
+def test_public_bucket():
+    # The new projects don't allow to make a bucket available to public, so
+    # for some tests we need to use the old main project for now.
+    original_value = os.environ['GOOGLE_CLOUD_PROJECT']
+    os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT']
+    bucket = None
+    while bucket is None or bucket.exists():
+        storage_client = storage.Client()
+        bucket_name = "storage-snippets-test-{}".format(uuid.uuid4())
+        bucket = storage_client.bucket(bucket_name)
+    storage_client.create_bucket(bucket)
+    yield bucket
+    bucket.delete(force=True)
+    # Set the value back.
+    os.environ['GOOGLE_CLOUD_PROJECT'] = original_value
+
+
 @pytest.fixture
 def test_blob(test_bucket):
     """Yields a blob that is deleted after the test completes."""
@@ -114,6 +132,15 @@ def test_blob(test_bucket):
     yield blob
 
 
+@pytest.fixture(scope="function")
+def test_public_blob(test_public_bucket):
+    """Yields a blob that is deleted after the test completes."""
+    bucket = test_public_bucket
+    blob = bucket.blob("storage_snippets_test_sigil-{}".format(uuid.uuid4()))
+    blob.upload_from_string("Hello, is it me you're looking for?")
+    yield blob
+
+
 @pytest.fixture
 def test_bucket_create():
     """Yields a bucket object that is deleted after the test completes."""
@@ -196,10 +223,11 @@ def test_delete_blob(test_blob):
     storage_delete_file.delete_blob(test_blob.bucket.name, test_blob.name)
 
 
-def test_make_blob_public(test_blob):
-    storage_make_public.make_blob_public(test_blob.bucket.name, test_blob.name)
+def test_make_blob_public(test_public_blob):
+    storage_make_public.make_blob_public(
+        test_public_blob.bucket.name, test_public_blob.name)
 
-    r = requests.get(test_blob.public_url)
+    r = requests.get(test_public_blob.public_url)
     assert r.text == "Hello, is it me you're looking for?"
 
 
@@ -342,11 +370,12 @@ def test_get_service_account(capsys):
     assert "@gs-project-accounts.iam.gserviceaccount.com" in out
 
 
-def test_download_public_file(test_blob):
-    storage_make_public.make_blob_public(test_blob.bucket.name, test_blob.name)
+def test_download_public_file(test_public_blob):
+    storage_make_public.make_blob_public(
+        test_public_blob.bucket.name, test_public_blob.name)
     with tempfile.NamedTemporaryFile() as dest_file:
         storage_download_public_file.download_public_file(
-            test_blob.bucket.name, test_blob.name, dest_file.name
+            test_public_blob.bucket.name, test_public_blob.name, dest_file.name
         )
 
         assert dest_file.read() == b"Hello, is it me you're looking for?"

storage/cloud-client/uniform_bucket_level_access_test.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import time
+import uuid
 
 from google.cloud import storage
 import pytest
@@ -25,14 +27,20 @@
 @pytest.fixture()
 def bucket():
     """Yields a bucket that is deleted after the test completes."""
+    # The new projects enforces uniform bucket level access, so
+    # we need to use the old main project for now.
+    original_value = os.environ['GOOGLE_CLOUD_PROJECT']
+    os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT']
     bucket = None
     while bucket is None or bucket.exists():
-        bucket_name = "uniform-bucket-level-access-{}".format(int(time.time()))
+        bucket_name = "uniform-bucket-level-access-{}".format(uuid.uuid4().hex)
         bucket = storage.Client().bucket(bucket_name)
     bucket.create()
     yield bucket
     time.sleep(3)
     bucket.delete(force=True)
+    # Set the value back.
+    os.environ['GOOGLE_CLOUD_PROJECT'] = original_value
 
 
 def test_get_uniform_bucket_level_access(bucket, capsys):