Skip to content

Commit c75abf2

Browse files
authored
Update docs (#1590)
1 parent f628fe7 commit c75abf2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+262
-496
lines changed

.gitbook.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,6 @@ redirects:
88
tutorial: ../examples/pytorch/text-generator/README.md
99
tutorial/realtime: ../examples/pytorch/text-generator/README.md
1010
tutorial/batch: ../examples/batch/image-classifier/README.md
11+
install: ./aws/install.md
12+
uninstall: ./aws/uninstall.md
13+
update: ./aws/update.md

cli/cluster/errors.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"net/url"
2222
"strings"
2323

24+
"github.com/cortexlabs/cortex/pkg/consts"
2425
"github.com/cortexlabs/cortex/pkg/lib/errors"
2526
"github.com/cortexlabs/cortex/pkg/lib/urls"
2627
)
@@ -61,8 +62,7 @@ func ErrorFailedToConnectOperator(originalError error, envName string, operatorU
6162
msg += fmt.Sprintf(" → otherwise you can ignore this message, and prevent it in the future with `cortex env delete %s`\n", envName)
6263
msg += "\nif you have a cluster running:\n"
6364
msg += fmt.Sprintf(" → run `cortex cluster info --configure-env %s` to update your environment (include `--config <cluster.yaml>` if you have a cluster configuration file)\n", envName)
64-
// CORTEX_VERSION_MINOR
65-
msg += " → if you set `operator_load_balancer_scheme: internal` in your cluster configuration file, your CLI must run from within a VPC that has access to your cluster's VPC (see https://docs.cortex.dev/v/master/guides/vpc-peering)\n"
65+
msg += fmt.Sprintf(" → if you set `operator_load_balancer_scheme: internal` in your cluster configuration file, your CLI must run from within a VPC that has access to your cluster's VPC (see https://docs.cortex.dev/v/%s/aws/vpc-peering)\n", consts.CortexVersionMinor)
6666
}
6767

6868
return errors.WithStack(&errors.Error{

cli/cmd/errors.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ func ErrorMissingAWSCredentials() error {
248248
func ErrorCredentialsInClusterConfig(cmd string, path string) error {
249249
return errors.WithStack(&errors.Error{
250250
Kind: ErrCredentialsInClusterConfig,
251-
Message: fmt.Sprintf("specifying credentials in the cluster configuration is no longer supported, please specify aws credentials using flags (e.g. cortex cluster %s --config %s --aws-key <AWS_ACCESS_KEY_ID> --aws-secret <AWS_SECRET_ACCESS_KEY>) or set environment variables; see https://docs.cortex.dev/v/%s/miscellaneous/security#iam-permissions for more information", cmd, path, consts.CortexVersionMinor),
251+
Message: fmt.Sprintf("specifying credentials in the cluster configuration is no longer supported, please specify aws credentials using flags (e.g. cortex cluster %s --config %s --aws-key <AWS_ACCESS_KEY_ID> --aws-secret <AWS_SECRET_ACCESS_KEY>) or set environment variables; see https://docs.cortex.dev/v/%s/aws/security#iam-permissions for more information", cmd, path, consts.CortexVersionMinor),
252252
})
253253
}
254254

cli/cmd/lib_aws_creds.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"os"
2222
"path/filepath"
2323

24+
"github.com/cortexlabs/cortex/pkg/consts"
2425
"github.com/cortexlabs/cortex/pkg/lib/aws"
2526
"github.com/cortexlabs/cortex/pkg/lib/errors"
2627
libjson "github.com/cortexlabs/cortex/pkg/lib/json"
@@ -68,7 +69,7 @@ func promptIfNotAdmin(awsClient *aws.Client, disallowPrompt bool) {
6869
}
6970

7071
if !awsClient.IsAdmin() {
71-
warningStr := fmt.Sprintf("warning: your IAM user%s does not have administrator access. This will likely prevent Cortex from installing correctly, so it is recommended to attach the AdministratorAccess policy to your IAM user (or to a group that your IAM user belongs to) via the AWS IAM console. If you'd like, you may provide separate credentials for your cluster to use after it's running (see https://docs.cortex.dev/miscellaneous/security for instructions).\n\n", accessKeyMsg)
72+
warningStr := fmt.Sprintf("warning: your IAM user%s does not have administrator access. This will likely prevent Cortex from installing correctly, so it is recommended to attach the AdministratorAccess policy to your IAM user (or to a group that your IAM user belongs to) via the AWS IAM console. If you'd like, you may provide separate credentials for your cluster to use after it's running (see https://docs.cortex.dev/v/%s/aws/security for instructions).\n\n", accessKeyMsg, consts.CortexVersionMinor)
7273
if disallowPrompt {
7374
fmt.Print(warningStr)
7475
} else {

cli/cmd/lib_cluster_config.go

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func readCachedClusterConfigFile(clusterConfig *clusterconfig.Config, filePath s
7070
func readUserClusterConfigFile(clusterConfig *clusterconfig.Config) error {
7171
errs := cr.ParseYAMLFile(clusterConfig, clusterconfig.UserValidation, _flagClusterConfig)
7272
if errors.HasError(errs) {
73-
return errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/cluster-management/config", consts.CortexVersionMinor))
73+
return errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/aws/install", consts.CortexVersionMinor))
7474
}
7575

7676
return nil
@@ -85,7 +85,7 @@ func getNewClusterAccessConfig(disallowPrompt bool) (*clusterconfig.AccessConfig
8585
if _flagClusterConfig != "" {
8686
errs := cr.ParseYAMLFile(accessConfig, clusterconfig.AccessValidation, _flagClusterConfig)
8787
if errors.HasError(errs) {
88-
return nil, errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/cluster-management/config", consts.CortexVersionMinor))
88+
return nil, errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/aws/install", consts.CortexVersionMinor))
8989
}
9090
}
9191

@@ -121,7 +121,7 @@ func getClusterAccessConfigWithCache(disallowPrompt bool) (*clusterconfig.Access
121121
if _flagClusterConfig != "" {
122122
errs := cr.ParseYAMLFile(accessConfig, clusterconfig.AccessValidation, _flagClusterConfig)
123123
if errors.HasError(errs) {
124-
return nil, errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/cluster-management/config", consts.CortexVersionMinor))
124+
return nil, errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/aws/install", consts.CortexVersionMinor))
125125
}
126126
}
127127

@@ -206,7 +206,7 @@ func getInstallClusterConfig(awsCreds AWSCredentials, accessConfig clusterconfig
206206

207207
err = clusterConfig.Validate(awsClient)
208208
if err != nil {
209-
err = errors.Append(err, fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/cluster-management/config", consts.CortexVersionMinor))
209+
err = errors.Append(err, fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/aws/install", consts.CortexVersionMinor))
210210
if _flagClusterConfig != "" {
211211
err = errors.Wrap(err, _flagClusterConfig)
212212
}
@@ -272,7 +272,7 @@ func getConfigureClusterConfig(cachedClusterConfig clusterconfig.Config, awsCred
272272

273273
err = userClusterConfig.Validate(awsClient)
274274
if err != nil {
275-
err = errors.Append(err, fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/cluster-management/config", consts.CortexVersionMinor))
275+
err = errors.Append(err, fmt.Sprintf("\n\ncluster configuration schema can be found here: https://docs.cortex.dev/v/%s/aws/install", consts.CortexVersionMinor))
276276
if _flagClusterConfig != "" {
277277
err = errors.Wrap(err, _flagClusterConfig)
278278
}
@@ -556,23 +556,23 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds A
556556
fmt.Printf("cortex will also create an s3 bucket (%s) and a cloudwatch log group (%s)%s\n\n", clusterConfig.Bucket, clusterConfig.ClusterName, privateSubnetMsg)
557557

558558
if clusterConfig.APIGatewaySetting == clusterconfig.NoneAPIGatewaySetting {
559-
fmt.Print("warning: you've disabled API Gateway cluster-wide, so APIs will not be able to create API Gateway endpoints (they will still be reachable via the API load balancer; see https://docs.cortex.dev/deployments/networking for more information)\n\n")
559+
fmt.Print(fmt.Sprintf("warning: you've disabled API Gateway cluster-wide, so APIs will not be able to create API Gateway endpoints (they will still be reachable via the API load balancer; see https://docs.cortex.dev/v/%s/aws/networking for more information)\n\n", consts.CortexVersionMinor))
560560
}
561561

562562
if clusterConfig.OperatorLoadBalancerScheme == clusterconfig.InternalLoadBalancerScheme {
563-
fmt.Print("warning: you've configured the operator load balancer to be internal; you must configure VPC Peering to connect your CLI to your cluster operator (see https://docs.cortex.dev/guides/vpc-peering)\n\n")
563+
fmt.Print(fmt.Sprintf("warning: you've configured the operator load balancer to be internal; you must configure VPC Peering to connect your CLI to your cluster operator (see https://docs.cortex.dev/v/%s/aws/vpc-peering)\n\n", consts.CortexVersionMinor))
564564
}
565565

566566
if isSpot && clusterConfig.SpotConfig.OnDemandBackup != nil && !*clusterConfig.SpotConfig.OnDemandBackup {
567567
if *clusterConfig.SpotConfig.OnDemandBaseCapacity == 0 && *clusterConfig.SpotConfig.OnDemandPercentageAboveBaseCapacity == 0 {
568-
fmt.Printf("warning: you've disabled on-demand instances (%s=0 and %s=0); spot instances are not guaranteed to be available so please take that into account for production clusters; see https://docs.cortex.dev/v/%s/cluster-management/spot-instances for more information\n\n", clusterconfig.OnDemandBaseCapacityKey, clusterconfig.OnDemandPercentageAboveBaseCapacityKey, consts.CortexVersionMinor)
568+
fmt.Printf("warning: you've disabled on-demand instances (%s=0 and %s=0); spot instances are not guaranteed to be available so please take that into account for production clusters; see https://docs.cortex.dev/v/%s/aws/spot for more information\n\n", clusterconfig.OnDemandBaseCapacityKey, clusterconfig.OnDemandPercentageAboveBaseCapacityKey, consts.CortexVersionMinor)
569569
} else {
570-
fmt.Printf("warning: you've enabled spot instances; spot instances are not guaranteed to be available so please take that into account for production clusters; see https://docs.cortex.dev/v/%s/cluster-management/spot-instances for more information\n\n", consts.CortexVersionMinor)
570+
fmt.Printf("warning: you've enabled spot instances; spot instances are not guaranteed to be available so please take that into account for production clusters; see https://docs.cortex.dev/v/%s/aws/spot for more information\n\n", consts.CortexVersionMinor)
571571
}
572572
}
573573

574574
if !disallowPrompt {
575-
exitMessage := fmt.Sprintf("cluster configuration can be modified via the cluster config file; see https://docs.cortex.dev/v/%s/cluster-management/config for more information", consts.CortexVersionMinor)
575+
exitMessage := fmt.Sprintf("cluster configuration can be modified via the cluster config file; see https://docs.cortex.dev/v/%s/aws/install for more information", consts.CortexVersionMinor)
576576
prompt.YesOrExit("would you like to continue?", "", exitMessage)
577577
}
578578
}
@@ -581,7 +581,7 @@ func confirmConfigureClusterConfig(clusterConfig clusterconfig.Config, awsCreds
581581
fmt.Println(clusterConfigConfirmationStr(clusterConfig, awsCreds, awsClient))
582582

583583
if !disallowPrompt {
584-
exitMessage := fmt.Sprintf("cluster configuration can be modified via the cluster config file; see https://docs.cortex.dev/v/%s/cluster-management/config for more information", consts.CortexVersionMinor)
584+
exitMessage := fmt.Sprintf("cluster configuration can be modified via the cluster config file; see https://docs.cortex.dev/v/%s/aws/install for more information", consts.CortexVersionMinor)
585585
prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s will be updated according to the configuration above, are you sure you want to continue?", clusterConfig.ClusterName, *clusterConfig.Region), "", exitMessage)
586586
}
587587
}

cli/local/api.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@ func UpdateAPI(apiConfig *userconfig.API, models []spec.CuratedModelResource, co
5656
fmt.Sprintf(
5757
"api %s was deployed using CLI version %s but the current CLI version is %s; "+
5858
"re-deploying %s with current CLI version %s might yield an unexpected outcome; any cached models won't be deleted\n\n"+
59-
"it is recommended to download version %s of the CLI from https://docs.cortex.dev/v/%s/install, delete the API using version %s of the CLI and then re-deploy the API using the latest version of the CLI\n\n"+
59+
"it is recommended to install version %s of the CLI (pip install cortex==%s), delete the API using version %s of the CLI, and then re-deploy the API using the latest version of the CLI\n\n"+
6060
"do you still want to re-deploy?",
61-
apiConfig.Name, incompatibleMinorVersion, consts.CortexVersionMinor, apiConfig.Name, consts.CortexVersionMinor, incompatibleMinorVersion, incompatibleMinorVersion, incompatibleMinorVersion),
61+
apiConfig.Name, incompatibleMinorVersion, consts.CortexVersionMinor, apiConfig.Name, consts.CortexVersionMinor, incompatibleMinorVersion, incompatibleVersion, incompatibleMinorVersion),
6262
"", "",
6363
)
6464
}

cli/local/delete.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,9 @@ func Delete(apiName string, keepCache, deleteForce bool) (schema.DeleteResponse,
4848
fmt.Sprintf(
4949
"api %s was deployed using CLI version %s but the current CLI version is %s; "+
5050
"deleting %s with current CLI version %s might lead to an unexpected state; any cached models won't be deleted\n\n"+
51-
"it is recommended to download version %s of the CLI from https://docs.cortex.dev/v/%s/install, delete the API using version %s of the CLI and then re-deploy the API using the latest version of the CLI\n\n"+
51+
"it is recommended to download version %s of the CLI (pip install cortex==%s), delete the API using version %s of the CLI, and then re-deploy the API using the latest version of the CLI\n\n"+
5252
"do you still want to delete?",
53-
apiName, incompatibleMinorVersion, consts.CortexVersionMinor, apiName, consts.CortexVersionMinor, incompatibleMinorVersion, incompatibleMinorVersion, incompatibleMinorVersion),
53+
apiName, incompatibleMinorVersion, consts.CortexVersionMinor, apiName, consts.CortexVersionMinor, incompatibleMinorVersion, incompatibleVersion, incompatibleMinorVersion),
5454
"", "",
5555
)
5656
}

cli/local/validations.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ import (
3939
"github.com/cortexlabs/cortex/pkg/types/userconfig"
4040
)
4141

42-
var _startingPort = 8890
42+
var _startingPort = 8889
4343

4444
type ProjectFiles struct {
4545
relFilePaths []string

docs/README.md

Lines changed: 0 additions & 1 deletion
This file was deleted.

docs/aws/credentials.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Credentials
2+
3+
_WARNING: you are on the master branch, please refer to the docs on the branch that matches your `cortex version`_
4+
5+
1. Follow this [tutorial](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key) to create an access key.
6+
1. Enable programmatic access for the IAM user, and attach the built-in `AdministratorAccess` policy to your IAM user.
7+
1. See [security](security.md) if you'd like to use less privileged credentials after spinning up your cluster.
File renamed without changes.

docs/aws/install.md

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# Install
2+
3+
_WARNING: you are on the master branch, please refer to the docs on the branch that matches your `cortex version`_
4+
5+
## Spin up Cortex on your AWS account
6+
7+
Make sure [Docker](https://docs.docker.com/install) is running on your machine.
8+
9+
If you're using GPUs, subscribe to the [EKS-optimized AMI with GPU Support](https://aws.amazon.com/marketplace/pp/B07GRHFXGM) before creating your cluster.
10+
11+
```bash
12+
# install the CLI
13+
pip install cortex
14+
15+
# spin up Cortex on your AWS account
16+
cortex cluster up # or: cortex cluster up --config cluster.yaml (see configuration options below)
17+
18+
# set the default environment
19+
cortex env default aws
20+
```
21+
22+
<!-- CORTEX_VERSION_MINOR -->
23+
Try the [tutorial](../../examples/pytorch/text-generator/README.md) or deploy one of our [examples](https://github.com/cortexlabs/cortex/tree/master/examples).
24+
25+
## Configure Cortex
26+
27+
<!-- CORTEX_VERSION_MINOR -->
28+
```yaml
29+
# cluster.yaml
30+
31+
# EKS cluster name
32+
cluster_name: cortex
33+
34+
# AWS region
35+
region: us-east-1
36+
37+
# list of availability zones for your region
38+
availability_zones: # default: 3 random availability zones in your region, e.g. [us-east-1a, us-east-1b, us-east-1c]
39+
40+
# instance type
41+
instance_type: m5.large
42+
43+
# minimum number of instances
44+
min_instances: 1
45+
46+
# maximum number of instances
47+
max_instances: 5
48+
49+
# disk storage size per instance (GB)
50+
instance_volume_size: 50
51+
52+
# instance volume type [gp2 | io1 | st1 | sc1]
53+
instance_volume_type: gp2
54+
55+
# instance volume iops (only applicable to io1)
56+
# instance_volume_iops: 3000
57+
58+
# subnet visibility [public (instances will have public IPs) | private (instances will not have public IPs)]
59+
subnet_visibility: public
60+
61+
# NAT gateway (required when using private subnets) [none | single | highly_available (a NAT gateway per availability zone)]
62+
nat_gateway: none
63+
64+
# API load balancer scheme [internet-facing | internal]
65+
api_load_balancer_scheme: internet-facing
66+
67+
# operator load balancer scheme [internet-facing | internal]
68+
# note: if using "internal", you must configure VPC Peering to connect your CLI to your cluster operator (https://docs.cortex.dev/v/master/aws/vpc-peering)
69+
operator_load_balancer_scheme: internet-facing
70+
71+
# API Gateway [public (API Gateway will be used by default, can be disabled per API) | none (API Gateway will be disabled for all APIs)]
72+
api_gateway: public
73+
74+
# additional tags to assign to AWS resources (all resources will automatically be tagged with cortex.dev/cluster-name: <cluster_name>)
75+
tags: # <string>: <string> map of key/value pairs
76+
77+
# enable spot instances
78+
spot: false
79+
80+
# SSL certificate ARN (only necessary when using a custom domain without API Gateway)
81+
ssl_certificate_arn:
82+
83+
# primary CIDR block for the cluster's VPC
84+
vpc_cidr: 192.168.0.0/16
85+
```
86+
87+
The docker images used by the Cortex cluster can also be overridden, although this is not common. They can be configured by adding any of these keys to your cluster configuration file (default values are shown):
88+
89+
<!-- CORTEX_VERSION_BRANCH_STABLE -->
90+
```yaml
91+
image_operator: quay.io/cortexlabs/operator:master
92+
image_manager: quay.io/cortexlabs/manager:master
93+
image_downloader: quay.io/cortexlabs/downloader:master
94+
image_request_monitor: quay.io/cortexlabs/request-monitor:master
95+
image_cluster_autoscaler: quay.io/cortexlabs/cluster-autoscaler:master
96+
image_metrics_server: quay.io/cortexlabs/metrics-server:master
97+
image_inferentia: quay.io/cortexlabs/inferentia:master
98+
image_neuron_rtd: quay.io/cortexlabs/neuron-rtd:master
99+
image_nvidia: quay.io/cortexlabs/nvidia:master
100+
image_fluentd: quay.io/cortexlabs/fluentd:master
101+
image_statsd: quay.io/cortexlabs/statsd:master
102+
image_istio_proxy: quay.io/cortexlabs/istio-proxy:master
103+
image_istio_pilot: quay.io/cortexlabs/istio-pilot:master
104+
```
105+
106+
The default docker images used for your Predictors are listed in the instructions for [system packages](../deployments/system-packages.md), and can be overridden in your [Realtime API configuration](../deployments/realtime-api/api-configuration.md) and in your [Batch API configuration](../deployments/batch-api/api-configuration.md).
107+
108+
## Advanced
109+
110+
* [Security](security.md)
111+
* [VPC peering](vpc-peering.md)
112+
* [Custom domain](custom-domain.md)
113+
* [REST API Gateway](rest-api-gateway.md)
114+
* [Spot instances](spot.md)
115+
* [SSH into instances](ssh.md)

0 commit comments

Comments
 (0)