RCBC-464: Improve stability of management tests

bin/jenkins/test-with-cbdyncluster

@@ -53,12 +53,13 @@ fi
 CLUSTER_ID=$(cbdyncluster allocate --num-nodes=3 --server-version=${CB_VERSION})
 SERVICES="kv,index,n1ql,fts"
 CB_BUCKET=default
-cbdyncluster setup ${CLUSTER_ID} \
+CLUSTER_ENTRY_POINT=$(cbdyncluster setup ${CLUSTER_ID} \
+    --ram-quota=2048 \
     --storage-mode=plasma \
     ${extra_options} \
     --node=${SERVICES} \
     --node=${SERVICES} \
-    --node=${SERVICES}
+    --node=${SERVICES})
 cbdyncluster add-bucket ${CLUSTER_ID} \
     --name default \
     --ram-quota 256 \
@@ -121,6 +122,7 @@ then
 fi
 export TEST_SERVER_VERSION=${CB_VERSION}
 export TEST_CONNECTION_STRING=$(cbdyncluster connstr ${CLUSTER_ID})
+export TEST_MANAGEMENT_ENDPOINT=${CLUSTER_ENTRY_POINT}
 set +e
 
 cat /proc/sys/kernel/core_pattern

test/bucket_manager_test.rb

@@ -54,6 +54,9 @@ def test_create_bucket_history_retention
           s.history_retention_duration = 600
         end
       )
+
+      env.consistency.wait_until_bucket_present(bucket_name)
+
       res = @bucket_manager.get_bucket(bucket_name)
 
       refute res.history_retention_collection_default
@@ -91,6 +94,9 @@ def test_update_bucket_history_retention
           s.ram_quota_mb = 1024
         end
       )
+
+      env.consistency.wait_until_bucket_present(bucket_name)
+
       res = @bucket_manager.get_bucket(bucket_name)
 
       assert res.history_retention_collection_default
@@ -125,6 +131,9 @@ def test_update_bucket_history_retention_unsupported
           s.storage_backend = :couchstore
         end
       )
+
+      env.consistency.wait_until_bucket_present(bucket_name)
+
       res = @bucket_manager.get_bucket(bucket_name)
 
       assert_nil res.history_retention_collection_default

test/collection_manager_test.rb

@@ -29,8 +29,9 @@ def create_magma_bucket
           s.ram_quota_mb = 1024
         end
       )
-      begin
-      end
+
+      env.consistency.wait_until_bucket_present(TEST_MAGMA_BUCKET_NAME)
+
       retry_for_duration(expected_errors: [Error::BucketNotFound]) do
         @magma_bucket = @cluster.bucket(TEST_MAGMA_BUCKET_NAME)
       end
@@ -57,6 +58,8 @@ def get_collection(scope_name, collection_name, mgr = nil)
     end
 
     def setup
+      skip("#{name}: The server does not support collections") unless env.server_version.supports_collections?
+
       connect
       @used_scopes = []
       @bucket = @cluster.bucket(env.bucket)
@@ -86,6 +89,7 @@ def teardown
     def test_create_scope
       scope_name = get_scope_name
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       refute_nil scope
@@ -94,11 +98,13 @@ def test_create_scope
     def test_drop_scope
       scope_name = get_scope_name
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       refute_nil scope
 
       @collection_manager.drop_scope(scope_name)
+      env.consistency.wait_until_scope_dropped(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       assert_nil scope
@@ -108,8 +114,11 @@ def test_create_collection
       coll_names = %w[coll-1 coll-2 coll-3]
       scope_name = get_scope_name
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
+
       coll_names.each do |coll_name|
         @collection_manager.create_collection(scope_name, coll_name)
+        env.consistency.wait_until_collection_present(env.bucket, scope_name, coll_name)
       end
       scope = get_scope(scope_name)
 
@@ -121,15 +130,20 @@ def test_drop_collection
       coll_names = %w[coll-1 coll-2 coll-3]
       scope_name = get_scope_name
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
+
       coll_names.each do |coll_name|
         @collection_manager.create_collection(scope_name, coll_name)
+        env.consistency.wait_until_collection_present(env.bucket, scope_name, coll_name)
       end
       scope = get_scope(scope_name)
 
       refute_nil scope
       assert_equal coll_names.sort, scope.collections.map(&:name).sort
 
       @collection_manager.drop_collection(scope_name, 'coll-1')
+      env.consistency.wait_until_collection_dropped(env.bucket, scope_name, 'coll-1')
+
       scope = get_scope(scope_name)
 
       refute_includes scope.collections.map(&:name), 'coll-1'
@@ -139,7 +153,10 @@ def test_create_collection_already_exists
       coll_name = 'coll-1'
       scope_name = get_scope_name
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
+
       @collection_manager.create_collection(scope_name, coll_name)
+      env.consistency.wait_until_collection_present(env.bucket, scope_name, coll_name)
 
       refute_nil get_collection(scope_name, coll_name)
 
@@ -170,6 +187,7 @@ def test_create_collection_scope_does_not_exist
     def test_create_scope_already_exists
       scope_name = get_scope_name
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
 
       refute_nil get_scope(scope_name)
 
@@ -190,6 +208,7 @@ def test_drop_collection_does_not_exist
       scope_name = get_scope_name
       coll_name = 'does-not-exist'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
 
       refute_nil get_scope(scope_name)
 
@@ -216,12 +235,14 @@ def test_create_collection_history_retention
       scope_name = get_scope_name
       collection_name = 'test-coll'
       @magma_collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(TEST_MAGMA_BUCKET_NAME, scope_name)
       scope = get_scope(scope_name, @magma_collection_manager)
 
       assert scope
 
       settings = Management::CreateCollectionSettings.new(history: true)
       @magma_collection_manager.create_collection(scope_name, collection_name, settings)
+      env.consistency.wait_until_collection_present(TEST_MAGMA_BUCKET_NAME, scope_name, collection_name)
 
       coll = get_collection(scope_name, collection_name, @magma_collection_manager)
 
@@ -239,12 +260,15 @@ def test_update_collection_history_retention
       scope_name = get_scope_name
       collection_name = 'test-coll'
       @magma_collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(TEST_MAGMA_BUCKET_NAME, scope_name)
+
       scope = get_scope(scope_name, @magma_collection_manager)
 
       assert scope
 
       settings = Management::CreateCollectionSettings.new(history: false)
       @magma_collection_manager.create_collection(scope_name, collection_name, settings)
+      env.consistency.wait_until_collection_present(TEST_MAGMA_BUCKET_NAME, scope_name, collection_name)
 
       coll = get_collection(scope_name, collection_name, @magma_collection_manager)
 
@@ -268,6 +292,8 @@ def test_create_collection_history_retention_unsupported
       scope_name = get_scope_name
       collection_name = 'test-coll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
+
       scope = get_scope(scope_name)
 
       assert scope
@@ -287,6 +313,8 @@ def test_update_collection_history_retention_unsupported
       scope_name = get_scope_name
       collection_name = 'test-coll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
+
       scope = get_scope(scope_name)
 
       assert scope
@@ -309,12 +337,14 @@ def test_create_collection_max_expiry
       scope_name = get_scope_name
       collection_name = 'testcoll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       assert scope
 
       settings = Management::CreateCollectionSettings.new(max_expiry: 5)
       @collection_manager.create_collection(scope_name, collection_name, settings)
+      env.consistency.wait_until_collection_present(env.bucket, scope_name, collection_name)
 
       coll_spec = get_collection(scope_name, collection_name)
 
@@ -348,13 +378,15 @@ def test_create_collection_max_expiry_no_expiry
       scope_name = get_scope_name
       collection_name = 'testcoll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       assert scope
 
       settings = Management::CreateCollectionSettings.new(max_expiry: -1)
 
       @collection_manager.create_collection(scope_name, collection_name, settings)
+      env.consistency.wait_until_collection_present(env.bucket, scope_name, collection_name)
 
       coll_spec = get_collection(scope_name, collection_name)
 
@@ -369,6 +401,7 @@ def test_create_collection_max_expiry_no_expiry_not_supported
       scope_name = get_scope_name
       collection_name = 'testcoll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       assert scope
@@ -384,6 +417,7 @@ def test_create_collection_max_expiry_invalid
       scope_name = get_scope_name
       collection_name = 'testcoll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       assert scope
@@ -404,12 +438,14 @@ def test_update_collection_max_expiry
       scope_name = get_scope_name
       collection_name = 'test-coll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       assert scope
 
       settings = Management::CreateCollectionSettings.new(max_expiry: 600)
       @collection_manager.create_collection(scope_name, collection_name, settings)
+      env.consistency.wait_until_collection_present(env.bucket, scope_name, collection_name)
 
       coll_spec = get_collection(scope_name, collection_name)
 
@@ -451,12 +487,14 @@ def test_update_collection_max_expiry_no_expiry
       scope_name = get_scope_name
       collection_name = 'testcoll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       assert scope
 
       settings = Management::CreateCollectionSettings.new(max_expiry: 600)
       @collection_manager.create_collection(scope_name, collection_name, settings)
+      env.consistency.wait_until_collection_present(env.bucket, scope_name, collection_name)
 
       coll_spec = get_collection(scope_name, collection_name)
 
@@ -475,18 +513,23 @@ def test_update_collection_max_expiry_no_expiry
 
     def test_update_collection_max_expiry_no_expiry_not_supported
       skip("#{name}: CAVES does not support update_collection") if use_caves?
+      unless env.server_version.supports_update_collection_max_expiry?
+        skip("#{name}: The server does not support update_collection with max_expiry")
+      end
       skip("#{name}: The #{Couchbase::Protostellar::NAME} protocol does not support update_collection") if env.protostellar?
       skip("#{name}: The server supports setting max_expiry to -1") if env.server_version.supports_collection_max_expiry_set_to_no_expiry?
 
       scope_name = get_scope_name
       collection_name = 'testcoll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       assert scope
 
       settings = Management::CreateCollectionSettings.new(max_expiry: 600)
       @collection_manager.create_collection(scope_name, collection_name, settings)
+      env.consistency.wait_until_collection_present(env.bucket, scope_name, collection_name)
 
       coll_spec = get_collection(scope_name, collection_name)
 
@@ -507,12 +550,14 @@ def test_update_collection_max_expiry_invalid
       scope_name = get_scope_name
       collection_name = 'testcoll'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
       scope = get_scope(scope_name)
 
       assert scope
 
       settings = Management::CreateCollectionSettings.new(max_expiry: 600)
       @collection_manager.create_collection(scope_name, collection_name, settings)
+      env.consistency.wait_until_collection_present(env.bucket, scope_name, collection_name)
 
       coll_spec = get_collection(scope_name, collection_name)
 
@@ -535,6 +580,7 @@ def test_update_collection_does_not_exist
       scope_name = get_scope_name
       coll_name = 'does-not-exist'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
 
       refute_nil get_scope(scope_name)
 
@@ -565,13 +611,16 @@ def test_create_collection_deprecated
       scope_name = get_scope_name
       coll_name = 'coll-1'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
+
       @collection_manager.create_collection(
         Management::CollectionSpec.new do |spec|
           spec.name = coll_name
           spec.scope_name = scope_name
         end,
         Management::Options::Collection::CreateCollection.new(timeout: 80_000)
       )
+      env.consistency.wait_until_collection_present(env.bucket, scope_name, coll_name)
 
       refute_nil get_collection(scope_name, coll_name)
     end
@@ -580,12 +629,15 @@ def test_drop_collection_deprecated
       scope_name = get_scope_name
       coll_name = 'coll-1'
       @collection_manager.create_scope(scope_name)
+      env.consistency.wait_until_scope_present(env.bucket, scope_name)
+
       @collection_manager.create_collection(
         Management::CollectionSpec.new do |spec|
           spec.name = coll_name
           spec.scope_name = scope_name
         end
       )
+      env.consistency.wait_until_collection_present(env.bucket, scope_name, coll_name)
 
       refute_nil get_collection(scope_name, coll_name)
 
@@ -596,6 +648,7 @@ def test_drop_collection_deprecated
         end,
         Management::Options::Collection::DropCollection.new(timeout: 80_000)
       )
+      env.consistency.wait_until_collection_dropped(env.bucket, scope_name, coll_name)
 
       assert_nil get_collection(scope_name, coll_name)
     end

test/collection_query_index_manager_test.rb

@@ -33,11 +33,15 @@ def setup
       collection_mgr = @bucket.collections
       collection_mgr.create_scope(@scope_name)
 
+      env.consistency.wait_until_scope_present(env.bucket, @scope_name)
+
       # Retry a few times in case the scope needs time to be created
       retry_for_duration(expected_errors: [Error::CouchbaseError]) do
         collection_mgr.create_collection(@scope_name, collection_name)
       end
 
+      env.consistency.wait_until_collection_present(env.bucket, @scope_name, collection_name)
+
       @collection = @bucket.scope(@scope_name).collection(collection_name)
 
       sleep(1)

test/query_index_manager_test.rb

@@ -28,6 +28,7 @@ def setup
           s.ram_quota_mb = 256
         end
       )
+      env.consistency.wait_until_bucket_present(@bucket_name)
       retry_for_duration(expected_errors: [Error::BucketNotFound]) do
         @bucket = @cluster.bucket('query-idx-test-bucket')
       end