Show API history in get command, add export API ID command (#1544)

Author: deliahu

cli/cluster/get.go

@@ -51,6 +51,20 @@ func GetAPI(operatorConfig OperatorConfig, apiName string) ([]schema.APIResponse
 	return apiRes, nil
 }
 
+func GetAPIByID(operatorConfig OperatorConfig, apiName string, apiID string) ([]schema.APIResponse, error) {
+	httpRes, err := HTTPGet(operatorConfig, "/get/"+apiName+"/"+apiID)
+	if err != nil {
+		return nil, err
+	}
+
+	var apiRes []schema.APIResponse
+	if err = json.Unmarshal(httpRes, &apiRes); err != nil {
+		return nil, errors.Wrap(err, "/get/"+apiName+"/"+apiID, string(httpRes))
+	}
+
+	return apiRes, nil
+}
+
 func GetJob(operatorConfig OperatorConfig, apiName string, jobID string) (schema.JobResponse, error) {
 	endpoint := path.Join("/batch", apiName)
 	httpRes, err := HTTPGet(operatorConfig, endpoint, map[string]string{"jobID": jobID})

cli/cmd/cluster.go

@@ -591,9 +591,9 @@ var _downCmd = &cobra.Command{
 }
 
 var _exportCmd = &cobra.Command{
-	Use:   "export",
-	Short: "download the code and configuration for all APIs deployed in a cluster",
-	Args:  cobra.NoArgs,
+	Use:   "export [API_NAME] [API_ID]",
+	Short: "download the code and configuration for APIs",
+	Args:  cobra.RangeArgs(0, 2),
 	Run: func(cmd *cobra.Command, args []string) {
 		telemetry.Event("cli.cluster.export")
 
@@ -649,14 +649,26 @@ var _exportCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		apisResponse, err := cluster.GetAPIs(operatorConfig)
-		if err != nil {
-			exit.Error(err)
-		}
-
-		if len(apisResponse) == 0 {
-			fmt.Println(fmt.Sprintf("no apis found in cluster named %s in %s", *accessConfig.ClusterName, *accessConfig.Region))
-			exit.Ok()
+		var apisResponse []schema.APIResponse
+		if len(args) == 0 {
+			apisResponse, err = cluster.GetAPIs(operatorConfig)
+			if err != nil {
+				exit.Error(err)
+			}
+			if len(apisResponse) == 0 {
+				fmt.Println(fmt.Sprintf("no apis found in your cluster named %s in %s", *accessConfig.ClusterName, *accessConfig.Region))
+				exit.Ok()
+			}
+		} else if len(args) == 1 {
+			apisResponse, err = cluster.GetAPI(operatorConfig, args[0])
+			if err != nil {
+				exit.Error(err)
+			}
+		} else if len(args) == 2 {
+			apisResponse, err = cluster.GetAPIByID(operatorConfig, args[0], args[1])
+			if err != nil {
+				exit.Error(err)
+			}
 		}
 
 		exportPath := fmt.Sprintf("export-%s-%s", *accessConfig.Region, *accessConfig.ClusterName)
@@ -667,7 +679,7 @@ var _exportCmd = &cobra.Command{
 		}
 
 		for _, apiResponse := range apisResponse {
-			baseDir := filepath.Join(exportPath, apiResponse.Spec.Name)
+			baseDir := filepath.Join(exportPath, apiResponse.Spec.Name, apiResponse.Spec.ID)
 
 			fmt.Println(fmt.Sprintf("exporting %s to %s", apiResponse.Spec.Name, baseDir))
 

cli/cmd/get.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/cortexlabs/cortex/cli/cluster"
 	"github.com/cortexlabs/cortex/cli/local"
@@ -28,10 +29,12 @@ import (
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
 	libjson "github.com/cortexlabs/cortex/pkg/lib/json"
+	"github.com/cortexlabs/cortex/pkg/lib/pointer"
 	"github.com/cortexlabs/cortex/pkg/lib/sets/strset"
 	s "github.com/cortexlabs/cortex/pkg/lib/strings"
 	"github.com/cortexlabs/cortex/pkg/lib/table"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
+	libtime "github.com/cortexlabs/cortex/pkg/lib/time"
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
 	"github.com/cortexlabs/cortex/pkg/types"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
@@ -63,6 +66,7 @@ func getInit() {
 	_getCmd.Flags().StringVarP(&_flagGetEnv, "env", "e", getDefaultEnv(_generalCommandType), "environment to use")
 	_getCmd.Flags().BoolVarP(&_flagWatch, "watch", "w", false, "re-run the command every 2 seconds")
 	_getCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.UserOutputTypeStrings(), "|")))
+	addVerboseFlag(_getCmd)
 }
 
 var _getCmd = &cobra.Command{
@@ -498,6 +502,23 @@ func getAPI(env cliconfig.Environment, apiName string) (string, error) {
 	return realtimeAPITable(apiRes, env)
 }
 
+func apiHistoryTable(apiVersions []schema.APIVersion) string {
+	t := table.Table{
+		Headers: []table.Header{
+			{Title: "api id"},
+			{Title: "last deployed"},
+		},
+	}
+
+	t.Rows = make([][]interface{}, len(apiVersions))
+	for i, apiVersion := range apiVersions {
+		lastUpdated := time.Unix(apiVersion.LastUpdated, 0)
+		t.Rows[i] = []interface{}{apiVersion.APIID, libtime.SinceStr(&lastUpdated)}
+	}
+
+	return t.MustFormat(&table.Opts{Sort: pointer.Bool(false)})
+}
+
 func titleStr(title string) string {
 	return "\n" + console.Bold(title) + "\n"
 }

cli/cmd/lib_batch_apis.go

@@ -131,9 +131,16 @@ func batchAPITable(batchAPI schema.APIResponse) string {
 		out += t.MustFormat()
 	}
 
-	out += "\n" + console.Bold("endpoint: ") + batchAPI.Endpoint
+	out += "\n" + console.Bold("endpoint: ") + batchAPI.Endpoint + "\n"
+
+	out += "\n" + apiHistoryTable(batchAPI.APIVersions)
+
+	if !_flagVerbose {
+		return out
+	}
+
+	out += titleStr("batch api configuration") + batchAPI.Spec.UserStr(types.AWSProviderType)
 
-	out += "\n" + titleStr("batch api configuration") + batchAPI.Spec.UserStr(types.AWSProviderType)
 	return out
 }
 

cli/cmd/lib_realtime_apis.go

@@ -74,6 +74,12 @@ func realtimeAPITable(realtimeAPI schema.APIResponse, env cliconfig.Environment)
 		out += "\n" + describeModelInput(realtimeAPI.Status, realtimeAPI.Spec.Predictor, realtimeAPI.Endpoint)
 	}
 
+	out += "\n" + apiHistoryTable(realtimeAPI.APIVersions)
+
+	if !_flagVerbose {
+		return out, nil
+	}
+
 	out += titleStr("configuration") + strings.TrimSpace(realtimeAPI.Spec.UserStr(env.Provider))
 
 	return out, nil

cli/cmd/lib_traffic_splitters.go

@@ -53,6 +53,12 @@ func trafficSplitterTable(trafficSplitter schema.APIResponse, env cliconfig.Envi
 	out += "\n" + console.Bold("endpoint: ") + trafficSplitter.Endpoint
 	out += fmt.Sprintf("\n%s curl %s -X POST -H \"Content-Type: application/json\" -d @sample.json\n", console.Bold("example curl:"), trafficSplitter.Endpoint)
 
+	out += "\n" + apiHistoryTable(trafficSplitter.APIVersions)
+
+	if !_flagVerbose {
+		return out, nil
+	}
+
 	out += titleStr("configuration") + strings.TrimSpace(trafficSplitter.Spec.UserStr(env.Provider))
 
 	return out, nil

cli/cmd/root.go

@@ -39,6 +39,7 @@ var (
 	_cmdStr string
 
 	_configFileExts = []string{"yaml", "yml"}
+	_flagVerbose    bool
 	_flagOutput     = flags.PrettyOutputType
 
 	_credentialsCacheDir string
@@ -203,6 +204,10 @@ func updateRootUsage() {
 	})
 }
 
+func addVerboseFlag(cmd *cobra.Command) {
+	cmd.Flags().BoolVarP(&_flagVerbose, "verbose", "v", false, "show additional information (only applies to pretty output format)")
+}
+
 func wasEnvFlagProvided(cmd *cobra.Command) bool {
 	envFlagProvided := false
 	cmd.Flags().VisitAll(func(flag *pflag.Flag) {

docs/miscellaneous/cli.md

@@ -58,6 +58,7 @@ Flags:
   -e, --env string      environment to use (default "local")
   -w, --watch           re-run the command every 2 seconds
   -o, --output string   output format: one of pretty|json (default "pretty")
+  -v, --verbose         show additional information (only applies to pretty output format)
   -h, --help            help for get
 ```
 
@@ -197,10 +198,10 @@ Flags:
 ### cluster export
 
 ```text
-download the code and configuration for all APIs deployed in a cluster
+download the code and configuration for APIs
 
 Usage:
-  cortex cluster export [flags]
+  cortex cluster export [API_NAME] [API_ID] [flags]
 
 Flags:
   -c, --config string       path to a cluster configuration file

pkg/lib/aws/s3.go

@@ -130,7 +130,7 @@ func IsValidS3aPath(s3aPath string) bool {
 // List all S3 objects that are "depth" levels or deeper than the given "s3Path".
 // Setting depth to 1 effectively translates to listing the objects one level or deeper than the given prefix (aka listing the directory contents).
 //
-// 1st returned value is the list of paths found at level <depth>.
+// 1st returned value is the list of paths found at level <depth> or deeper.
 // 2nd returned value is the list of paths found at all levels.
 func (c *Client) GetNLevelsDeepFromS3Path(s3Path string, depth int, includeDirObjects bool, maxResults *int64) ([]string, []string, error) {
 	paths := strset.New()
@@ -637,6 +637,31 @@ func (c *Client) ListS3PathDir(s3DirPath string, includeDirObjects bool, maxResu
 	return c.ListS3PathPrefix(s3Path, includeDirObjects, maxResults)
 }
 
+// This behaves like you'd expect `ls` to behave on a local file system
+// "directory" names will be returned even if S3 directory objects don't exist
+func (c *Client) ListS3DirOneLevel(bucket string, s3Dir string, maxResults *int64) ([]string, error) {
+	s3Dir = s.EnsureSuffix(s3Dir, "/")
+
+	allNames := strset.New()
+
+	err := c.S3Iterator(bucket, s3Dir, true, nil, func(object *s3.Object) (bool, error) {
+		relativePath := strings.TrimPrefix(*object.Key, s3Dir)
+		oneLevelPath := strings.Split(relativePath, "/")[0]
+		allNames.Add(oneLevelPath)
+
+		if maxResults != nil && int64(len(allNames)) >= *maxResults {
+			return false, nil
+		}
+		return true, nil
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, S3Path(bucket, s3Dir))
+	}
+
+	return allNames.SliceSorted(), nil
+}
+
 func (c *Client) ListS3Prefix(bucket string, prefix string, includeDirObjects bool, maxResults *int64) ([]*s3.Object, error) {
 	var allObjects []*s3.Object
 

pkg/lib/sets/strset/strset.go

@@ -190,7 +190,7 @@ func (s Set) Slice() []string {
 	return v
 }
 
-// List returns a sorted slice of all items.
+// List returns a sorted slice of all items (a to z).
 func (s Set) SliceSorted() []string {
 	v := s.Slice()
 	sort.Strings(v)

pkg/lib/sets/strset/threadsafe/strset.go

@@ -209,7 +209,7 @@ func (s *Set) Slice() []string {
 	return s.s.Slice()
 }
 
-// List returns a sorted slice of all items.
+// List returns a sorted slice of all items (a to z).
 func (s *Set) SliceSorted() []string {
 	s.RLock()
 	defer s.RUnlock()

pkg/operator/endpoints/get.go

@@ -44,3 +44,16 @@ func GetAPI(w http.ResponseWriter, r *http.Request) {
 
 	respond(w, response)
 }
+
+func GetAPIByID(w http.ResponseWriter, r *http.Request) {
+	apiName := mux.Vars(r)["apiName"]
+	apiID := mux.Vars(r)["apiID"]
+
+	response, err := resources.GetAPIByID(apiName, apiID)
+	if err != nil {
+		respondError(w, r, err)
+		return
+	}
+
+	respond(w, response)
+}

pkg/operator/main.go

@@ -87,6 +87,7 @@ func main() {
 	routerWithAuth.HandleFunc("/delete/{apiName}", endpoints.Delete).Methods("DELETE")
 	routerWithAuth.HandleFunc("/get", endpoints.GetAPIs).Methods("GET")
 	routerWithAuth.HandleFunc("/get/{apiName}", endpoints.GetAPI).Methods("GET")
+	routerWithAuth.HandleFunc("/get/{apiName}/{apiID}", endpoints.GetAPIByID).Methods("GET")
 	routerWithAuth.HandleFunc("/logs/{apiName}", endpoints.ReadLogs)
 
 	log.Print("Running on port " + _operatorPortStr)

pkg/operator/operator/deployed_resource.go

@@ -25,3 +25,7 @@ type DeployedResource struct {
 	userconfig.Resource
 	VirtualService *istioclientnetworking.VirtualService
 }
+
+func (deployedResourced *DeployedResource) ID() string {
+	return deployedResourced.VirtualService.Labels["apiID"]
+}

pkg/operator/resources/errors.go

@@ -29,6 +29,7 @@ import (
 const (
 	ErrOperationIsOnlySupportedForKind  = "resources.operation_is_only_supported_for_kind"
 	ErrAPINotDeployed                   = "resources.api_not_deployed"
+	ErrAPIIDNotFound                    = "resources.api_id_not_found"
 	ErrCannotChangeTypeOfDeployedAPI    = "resources.cannot_change_kind_of_deployed_api"
 	ErrNoAvailableNodeComputeLimit      = "resources.no_available_node_compute_limit"
 	ErrJobIDRequired                    = "resources.job_id_required"
@@ -58,6 +59,13 @@ func ErrorAPINotDeployed(apiName string) error {
 	})
 }
 
+func ErrorAPIIDNotFound(apiName string, apiID string) error {
+	return errors.WithStack(&errors.Error{
+		Kind:    ErrAPIIDNotFound,
+		Message: fmt.Sprintf("%s with id %s has never been deployed", apiName, apiID),
+	})
+}
+
 func ErrorCannotChangeKindOfDeployedAPI(name string, newKind, prevKind userconfig.Kind) error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrCannotChangeTypeOfDeployedAPI,