[secrets] Add secrets download and upload (#1720)

## Summary

Adds

```
devbox secrets download
devbox secrets upload
```

Also, updated envsec dependency that has a simpler API.

## How was it tested?

```
devbox secrets download foo.json
# modified foo.json
devbox secrets upload foo.json
```

Author: mikeland73

go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
 	github.com/wk8/go-ordered-map/v2 v2.1.8
 	github.com/zealic/go2node v0.1.0
-	go.jetpack.io/envsec v0.0.16-0.20240109233012-3e97a7fe973f
+	go.jetpack.io/envsec v0.0.16-0.20240111222345-e1fd0e1204ca
 	go.jetpack.io/pkg v0.0.0-20240108193620-a28b84329d15
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc
 	golang.org/x/mod v0.14.0

go.sum

@@ -378,8 +378,8 @@ github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBU
 github.com/zaffka/mongodb-boltdb-mock v0.0.0-20221014194232-b4bb03fbe3a0/go.mod h1:GsDD1qsG+86MeeCG7ndi6Ei3iGthKL3wQ7PTFigDfNY=
 github.com/zealic/go2node v0.1.0 h1:ofxpve08cmLJBwFdI0lPCk9jfwGWOSD+s6216x0oAaA=
 github.com/zealic/go2node v0.1.0/go.mod h1:GrkFr+HctXwP7vzcU9RsgtAeJjTQ6Ud0IPCQAqpTfBg=
-go.jetpack.io/envsec v0.0.16-0.20240109233012-3e97a7fe973f h1:ELgyeQw+eZn0BSSlhLnBkC7wTrYbluTsYx6vPa0YTds=
-go.jetpack.io/envsec v0.0.16-0.20240109233012-3e97a7fe973f/go.mod h1:8U0fUaXMaoNRqz8J7VaV8kP7N7hayJ9tqOd9OS0epEQ=
+go.jetpack.io/envsec v0.0.16-0.20240111222345-e1fd0e1204ca h1:7Ocnr+mVZTCG8MHlcAdW5d8ir7eqZTmD8S/aEskjXWk=
+go.jetpack.io/envsec v0.0.16-0.20240111222345-e1fd0e1204ca/go.mod h1:kCgRGNSHU5AgQXQGUenohPPDoUd87SRL00uOzQsa+Q8=
 go.jetpack.io/pkg v0.0.0-20240108193620-a28b84329d15 h1:ztX3CydpNKLePPMRmWgdi4HcxE/AXY6HViOJOmmYNiA=
 go.jetpack.io/pkg v0.0.0-20240108193620-a28b84329d15/go.mod h1:kGUL8aZ7ddvoGro0AQxXos9GKn5Qw0J18qW7d5FP4Ws=
 go.jetpack.io/typeid v1.0.0 h1:8gQ+iYGdyiQ0Pr40ydSB/PzMOIwlXX5DTojp1CBeSPQ=

internal/boxcli/secrets.go

@@ -16,6 +16,19 @@ type secretsFlags struct {
 	config configFlags
 }
 
+func (f *secretsFlags) envsec(cmd *cobra.Command) (*envsec.Envsec, error) {
+	box, err := devbox.Open(&devopt.Opts{
+		Dir:         f.config.path,
+		Environment: f.config.environment,
+		Stderr:      cmd.ErrOrStderr(),
+	})
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return box.Secrets(cmd.Context())
+}
+
 type secretsInitCmdFlags struct {
 	force bool
 }
@@ -25,6 +38,14 @@ type secretsListFlags struct {
 	format string
 }
 
+type secretsDownloadFlags struct {
+	format string
+}
+
+type secretsUploadFlags struct {
+	format string
+}
+
 func secretsCmd() *cobra.Command {
 	flags := &secretsFlags{}
 	cmd := &cobra.Command{
@@ -33,10 +54,12 @@ func secretsCmd() *cobra.Command {
 		Short:             "Interact with devbox secrets in jetpack cloud.",
 		PersistentPreRunE: ensureNixInstalled,
 	}
+	cmd.AddCommand(secretsDownloadCmd(flags))
 	cmd.AddCommand(secretsInitCmd(flags))
 	cmd.AddCommand(secretsListCmd(flags))
 	cmd.AddCommand(secretsRemoveCmd(flags))
 	cmd.AddCommand(secretsSetCmd(flags))
+	cmd.AddCommand(secretsUploadCmd(flags))
 	cmd.Hidden = true
 
 	flags.config.registerPersistent(cmd)
@@ -76,27 +99,12 @@ func secretsSetCmd(flags *secretsFlags) *cobra.Command {
 			return envsec.ValidateSetArgs(args)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := cmd.Context()
-			box, err := devbox.Open(&devopt.Opts{
-				Dir:         flags.config.path,
-				Environment: flags.config.environment,
-				Stderr:      cmd.ErrOrStderr(),
-			})
-			if err != nil {
-				return errors.WithStack(err)
-			}
-
-			secrets, err := box.Secrets(ctx)
-			if err != nil {
-				return errors.WithStack(err)
-			}
-
-			envID, err := secrets.EnvID()
+			secrets, err := flags.envsec(cmd)
 			if err != nil {
 				return errors.WithStack(err)
 			}
 
-			return secrets.SetFromArgs(ctx, envID, args)
+			return secrets.SetFromArgs(cmd.Context(), args)
 		},
 	}
 }
@@ -108,27 +116,12 @@ func secretsRemoveCmd(flags *secretsFlags) *cobra.Command {
 		Aliases: []string{"rm"},
 		Args:    cobra.MinimumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := cmd.Context()
-			box, err := devbox.Open(&devopt.Opts{
-				Dir:         flags.config.path,
-				Environment: flags.config.environment,
-				Stderr:      cmd.ErrOrStderr(),
-			})
-			if err != nil {
-				return errors.WithStack(err)
-			}
-
-			secrets, err := box.Secrets(ctx)
-			if err != nil {
-				return errors.WithStack(err)
-			}
-
-			envID, err := secrets.EnvID()
+			secrets, err := flags.envsec(cmd)
 			if err != nil {
 				return errors.WithStack(err)
 			}
 
-			return secrets.DeleteAll(ctx, envID, args...)
+			return secrets.DeleteAll(cmd.Context(), args...)
 		},
 	}
 }
@@ -141,33 +134,18 @@ func secretsListCmd(commonFlags *secretsFlags) *cobra.Command {
 		Short:   "List all secrets",
 		Args:    cobra.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := cmd.Context()
-			box, err := devbox.Open(&devopt.Opts{
-				Dir:         commonFlags.config.path,
-				Environment: commonFlags.config.environment,
-				Stderr:      cmd.ErrOrStderr(),
-			})
+			secrets, err := commonFlags.envsec(cmd)
 			if err != nil {
 				return errors.WithStack(err)
 			}
 
-			secrets, err := box.Secrets(ctx)
-			if err != nil {
-				return errors.WithStack(err)
-			}
-
-			envID, err := secrets.EnvID()
-			if err != nil {
-				return errors.WithStack(err)
-			}
-
-			vars, err := secrets.List(ctx, envID)
+			vars, err := secrets.List(cmd.Context())
 			if err != nil {
 				return err
 			}
 
-			return envsec.PrintEnvVars(
-				vars, cmd.OutOrStdout(), flags.show, flags.format)
+			return envsec.PrintEnvVar(
+				cmd.OutOrStdout(), secrets.EnvID, vars, flags.show, flags.format)
 		},
 	}
 
@@ -188,6 +166,60 @@ func secretsListCmd(commonFlags *secretsFlags) *cobra.Command {
 	return cmd
 }
 
+func secretsDownloadCmd(commonFlags *secretsFlags) *cobra.Command {
+	flags := secretsDownloadFlags{}
+	command := &cobra.Command{
+		Use:   "download <file1>",
+		Short: "Download environment variables into the specified file",
+		Args:  cobra.ExactArgs(1),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return envsec.ValidateFormat(flags.format)
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			secrets, err := commonFlags.envsec(cmd)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			return secrets.Download(cmd.Context(), args[0], flags.format)
+		},
+	}
+
+	command.Flags().StringVarP(
+		&flags.format, "format", "f", "", "file format: dotenv or json")
+
+	return command
+}
+
+func secretsUploadCmd(commonFlags *secretsFlags) *cobra.Command {
+	flags := &secretsUploadFlags{}
+	command := &cobra.Command{
+		Use:   "upload <file1> [<fileN>]...",
+		Short: "Upload variables defined in one or more .env files.",
+		Args:  cobra.MinimumNArgs(1),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return envsec.ValidateFormat(flags.format)
+		},
+		RunE: func(cmd *cobra.Command, paths []string) error {
+			secrets, err := commonFlags.envsec(cmd)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			return secrets.Upload(cmd.Context(), paths, flags.format)
+		},
+	}
+
+	command.Flags().StringVarP(
+		&flags.format, "format", "f", "", "File format: dotenv or json")
+
+	return command
+}
+
 func secretsInitFunc(
 	cmd *cobra.Command,
 	flags secretsInitCmdFlags,
@@ -201,10 +233,10 @@ func secretsInitFunc(
 	if err != nil {
 		return errors.WithStack(err)
 	}
-	secrets, err := box.Secrets(ctx)
-	if err != nil {
-		return errors.WithStack(err)
-	}
+
+	// devbox.Secrets() by default assumes project is initialized (and shows
+	// error if not). So we use UninitializedSecrets() here instead.
+	secrets := box.UninitializedSecrets(ctx)
 
 	if _, err := secrets.ProjectConfig(); err == nil &&
 		box.Config().EnvFrom != "jetpack-cloud" {

internal/devbox/devbox.go

@@ -1111,20 +1111,16 @@ func (d *Devbox) configEnvs(
 		if err != nil {
 			return nil, err
 		}
-		envID, err := secrets.EnvID()
-		if err != nil {
-			return nil, err
-		}
 
-		cloudSecrets, err := secrets.List(ctx, envID)
+		cloudSecrets, err := secrets.List(ctx)
 		if err != nil {
 			ux.Fwarning(
 				os.Stderr,
 				"Error reading secrets from jetpack cloud: %s\n\n",
 				err,
 			)
 		} else {
-			for _, secret := range cloudSecrets[envID] {
+			for _, secret := range cloudSecrets {
 				env[secret.Name] = secret.Value
 			}
 		}

internal/devbox/secrets.go

@@ -9,41 +9,37 @@ import (
 	"go.jetpack.io/pkg/envvar"
 )
 
-type secrets struct {
-	envsec.Envsec
-	EnvName string
-}
-
-func (d *Devbox) Secrets(ctx context.Context) (*secrets, error) {
-	envsecInstance := envsec.Envsec{
+func (d *Devbox) UninitializedSecrets(ctx context.Context) *envsec.Envsec {
+	return &envsec.Envsec{
 		APIHost: build.JetpackAPIHost(),
 		Auth: envsec.AuthConfig{
 			ClientID: envvar.Get("ENVSEC_CLIENT_ID", build.ClientID()),
 			Issuer:   envvar.Get("ENVSEC_ISSUER", build.Issuer()),
 		},
 		IsDev:      build.IsDev,
 		Stderr:     d.stderr,
+		Store:      &jetstore.JetpackAPIStore{},
 		WorkingDir: d.ProjectDir(),
 	}
-
-	store := &jetstore.JetpackAPIStore{}
-	if err := envsecInstance.SetStore(ctx, store); err != nil {
-		return nil, err
-	}
-	return &secrets{
-		Envsec:  envsecInstance,
-		EnvName: d.environment,
-	}, nil
 }
 
-func (s *secrets) EnvID() (envsec.EnvID, error) {
-	project, err := s.ProjectConfig()
+func (d *Devbox) Secrets(ctx context.Context) (*envsec.Envsec, error) {
+	envsecInstance := d.UninitializedSecrets(ctx)
+
+	project, err := envsecInstance.ProjectConfig()
 	if err != nil {
-		return envsec.EnvID{}, err
+		return nil, err
 	}
-	return envsec.EnvID{
-		EnvName:   s.EnvName,
-		ProjectID: project.ProjectID.String(),
+
+	envsecInstance.EnvID = envsec.EnvID{
+		EnvName:   d.environment,
 		OrgID:     project.OrgID.String(),
-	}, nil
+		ProjectID: project.ProjectID.String(),
+	}
+
+	if _, err := envsecInstance.InitForUser(ctx); err != nil {
+		return nil, err
+	}
+
+	return envsecInstance, nil
 }