Fix changes brought in by #1544 + others

Author: RobertLucian

cli/cmd/gcp_cluster.go

@@ -83,13 +83,9 @@ func upGCP(gcpPath string) {
 	}
 
 	GCP := &gcp.Client{}
-	if exists, err := GCP.BucketExists(gcpConfig.Bucket); err != nil {
+	err = GCP.CreateBucket(gcpConfig.Bucket, gcpConfig.Project, true)
+	if err != nil {
 		exit.Error(err)
-	} else if !exists {
-		err = GCP.CreateBucket(gcpConfig.Bucket, gcpConfig.Project)
-		if err != nil {
-			exit.Error(err)
-		}
 	}
 
 	fmt.Print("￮ spinning up a cluster .")

pkg/lib/gcp/errors.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2020 Cortex Labs, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcp
+
+import (
+	"github.com/cortexlabs/cortex/pkg/lib/errors"
+	"github.com/cortexlabs/cortex/pkg/lib/pointer"
+	"google.golang.org/api/googleapi"
+)
+
+const (
+	YouAlreadyOwnThisBucketErrorMessage = "You already own this bucket. Please select another name."
+	InvalidBucketNameErrorMessage       = "Sorry, that name is not available. Please try a different one."
+)
+
+func IsGCPError(err error) bool {
+	_, ok := errors.CauseOrSelf(err).(*googleapi.Error)
+	return ok
+}
+
+func IsErrCode(err error, errorCode int, errorMessage *string) bool {
+	gcpError, ok := errors.CauseOrSelf(err).(*googleapi.Error)
+	if !ok {
+		return false
+	}
+	if gcpError.Code == errorCode {
+		if errorMessage != nil && gcpError.Message == *errorMessage {
+			return true
+		}
+		if errorMessage != nil && gcpError.Message != *errorMessage {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+func DoesBucketAlreadyExistError(err error) bool {
+	return IsErrCode(err, 409, pointer.String(YouAlreadyOwnThisBucketErrorMessage))
+}
+
+func IsInvalidBucketNameError(err error) bool {
+	return IsErrCode(err, 409, pointer.String(InvalidBucketNameErrorMessage))
+}

pkg/lib/gcp/gcs.go

@@ -21,35 +21,38 @@ import (
 	"context"
 	"encoding/json"
 	"path/filepath"
+	"strings"
 
 	"cloud.google.com/go/storage"
+	"github.com/cortexlabs/cortex/pkg/lib/debug"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
+	"github.com/cortexlabs/cortex/pkg/lib/sets/strset"
 	s "github.com/cortexlabs/cortex/pkg/lib/strings"
+	"google.golang.org/api/googleapi"
 	"google.golang.org/api/iterator"
 )
 
 func GCSPath(bucket string, key string) string {
 	return "gs://" + filepath.Join(bucket, key)
 }
 
-func (c *Client) BucketExists(bucket string) (bool, error) {
-	gcsClient, err := c.GCS()
-	if err != nil {
-		return false, err
-	}
-	_, err = gcsClient.Bucket(bucket).Attrs(context.Background())
-	// TODO fix this
-	return err == nil, nil
-}
-
-func (c *Client) CreateBucket(bucket, projectID string) error {
+func (c *Client) CreateBucket(bucket, projectID string, ignoreErrorIfBucketExists bool) error {
 	gcsClient, err := c.GCS()
 	if err != nil {
 		return err
 	}
 	err = gcsClient.Bucket(bucket).Create(context.Background(), projectID, nil)
 	if err != nil {
-		return err
+		if e, ok := err.(*googleapi.Error); ok {
+			debug.Ppg(e)
+		}
+		if DoesBucketAlreadyExistError(err) {
+			if !ignoreErrorIfBucketExists {
+				return err
+			}
+		} else {
+			return err
+		}
 	}
 	return nil
 }
@@ -162,3 +165,38 @@ func (c *Client) ReadBytesFromGCS(bucket string, key string) ([]byte, error) {
 	}
 	return buf.Bytes(), nil
 }
+
+func (c *Client) ListGCSDirOneLevel(bucket string, gcsDir string, maxResults *int64) ([]string, error) {
+	gcsClient, err := c.GCS()
+	if err != nil {
+		return nil, err
+	}
+
+	gcsDir = s.EnsureSuffix(gcsDir, "/")
+
+	objectIterator := gcsClient.Bucket(bucket).Objects(context.Background(), &storage.Query{
+		Prefix: gcsDir,
+	})
+
+	allNames := strset.New()
+
+	for {
+		attrs, err := objectIterator.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		relativePath := strings.TrimPrefix(attrs.Name, gcsDir)
+		oneLevelPath := strings.Split(relativePath, "/")[0]
+		allNames.Add(oneLevelPath)
+
+		if maxResults != nil && int64(len(allNames)) >= *maxResults {
+			break
+		}
+	}
+
+	return allNames.SliceSorted(), nil
+}

pkg/operator/resources/resources.go

@@ -407,10 +407,20 @@ func GetAPIByID(apiName string, apiID string) ([]schema.APIResponse, error) {
 
 func getPastAPIDeploys(apiName string) ([]schema.APIVersion, error) {
 	var apiVersions []schema.APIVersion
+	var apiIDs []string
+	var err error
 
-	apiIDs, err := config.AWS.ListS3DirOneLevel(config.Cluster.Bucket, spec.KeysPrefix(apiName, config.Cluster.ClusterName), pointer.Int64(10))
-	if err != nil {
-		return nil, err
+	if config.Provider == types.AWSProviderType {
+		apiIDs, err = config.AWS.ListS3DirOneLevel(config.Cluster.Bucket, spec.KeysPrefix(apiName, config.Cluster.ClusterName), pointer.Int64(10))
+		if err != nil {
+			return nil, err
+		}
+	}
+	if config.Provider == types.GCPProviderType {
+		apiIDs, err = config.GCP.ListGCSDirOneLevel(config.GCPCluster.Bucket, spec.KeysPrefix(apiName, config.GCPCluster.ClusterName), pointer.Int64(10))
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	for _, apiID := range apiIDs {