Merge pull request #140 from droot/dep-manifest-upgrade

preserve user dependencies during 'update vendor'

Author: Phillip Wittrock

cmd/kubebuilder/initproject/dep_manifest.go

@@ -16,6 +16,23 @@ limitations under the License.
 
 package initproject
 
+const depManifestHeader = `
+# Users add deps lines here
+
+[prune]
+  go-tests = true
+  #unused-packages = true
+
+# Note: Stanzas below are generated by Kubebuilder and may be rewritten when
+# upgrading kubebuilder versions.
+`
+
+// depManifestKBMarker acts as a separater between the user managed dependencies
+// and KB generated dependencies. Content above this marker is user managed and
+// needs to be preserved across 'vendor update' operations. Content below this
+// marker is generated by KB and will be updated by KB during 'vendor update'.
+const depManifestKBMarker = `# DO NOT MODIFY BELOW THIS LINE.`
+
 // template for dep's manifest file (Gopkg.toml). This is generated using
 // scripts/generate_dep_manifest.sh scripts.
 const depManifestOverride = `

cmd/kubebuilder/initproject/vendor.go

@@ -17,10 +17,16 @@ limitations under the License.
 package initproject
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
+	"html/template"
+	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
+	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -49,30 +55,17 @@ func RunVendorInstall(cmd *cobra.Command, args []string) {
 	if !depExists() {
 		log.Fatalf("Dep is not installed. Follow steps at: https://golang.github.io/dep/docs/installation.html")
 	}
-	var backupFilename string
 	if Update {
-		backupFilename = fmt.Sprintf("%s.bkp", depManifestFile)
-		if err := os.Rename(depManifestFile, backupFilename); err != nil {
-			fmt.Printf("Error renaming existing Gopkg.toml file: %v \n", err)
-			return
+		if err := updateDepManifest(); err != nil {
+			log.Fatalf("error upgrading the dep manifest (Gopkg.toml): %v", err)
 		}
+	} else {
+		createNewDepManifest()
 	}
-	depTmplArgs := map[string]string{
-		"Version": version.GetVersion().KubeBuilderVersion,
-	}
-	util.Write(depManifestFile, "dep-manifest-file", depManifestTmpl, depTmplArgs)
 	if err := runDepEnsure(); err != nil {
 		fmt.Printf("Error running 'dep ensure': %v\n", err)
-		fmt.Printf("Previous Gopkg.toml file has been saved at '%s'\n", backupFilename)
 		return
 	}
-	if Update && backupFilename != "" {
-		err := os.Remove(backupFilename)
-		if err != nil {
-			fmt.Printf("Warning: failed to remove backup file: %s", backupFilename)
-		}
-	}
-	return
 }
 
 func runDepEnsure() error {
@@ -92,16 +85,94 @@ func depExists() bool {
 	return err == nil
 }
 
-var depManifestTmpl = fmt.Sprintf("%s\n%s", depManifestHeader, depManifestOverride)
+func createNewDepManifest() {
+	depTmplArgs := map[string]string{
+		"Version": version.GetVersion().KubeBuilderVersion,
+	}
+	depManifestTmpl := fmt.Sprintf("%s\n%s\n%s", depManifestHeader, depManifestKBMarker, depManifestOverride)
+	util.Write(depManifestFile, "dep-manifest-file", depManifestTmpl, depTmplArgs)
+}
+
+// updateDepManifest updates the existing dep manifest with newer dependencies.
+// dep manifest update workflow:
+// Try to read user managed dep manifest section. If success, then append the
+// user managed dep with KB managed section and update the dep Manifest.
+func updateDepManifest() error {
+	// open the existing dep manifest.
+	f, err := os.Open(depManifestFile)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// try to read content till the dep marker
+	userDeps, foundKBMarker, err := tryReadingUserDeps(f)
+	if err != nil {
+		return err
+	}
+
+	if !foundKBMarker {
+		// depManifest file or abort the operation here.
+		// for now, aborting.
+		log.Fatalf(`
+Failed to upgrade the dep manifest (Gopkg.toml) file. It seems that the dep manifest
+is not being managed by Kubebuilder. You can run the command with --overwrite-dep-manifest 
+flag if you want to re-initialize the dep manifest file. 
+`)
+	}
+
+	b := bytes.NewBufferString(userDeps)
+	err = addKubeBuilderDeps(b)
+	if err != nil {
+		return err
+	}
+
+	tmpfile, err := ioutil.TempFile(".", "dep")
+	if err != nil {
+		return err
+	}
 
-const depManifestHeader = `
-# Users add deps lines here
+	defer os.Remove(tmpfile.Name()) // clean up
 
-[prune]
-  go-tests = true
-  #unused-packages = true
+	_, err = tmpfile.Write(b.Bytes())
+	if err != nil {
+		return err
+	}
+	err = tmpfile.Close()
+	if err != nil {
+		return err
+	}
 
-# Note: Stanzas below are generated by Kubebuilder and may be rewritten when
-# upgrading kubebuilder versions.
-# DO NOT MODIFY BELOW THIS LINE.
-`
+	err = os.Rename(tmpfile.Name(), depManifestFile)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func tryReadingUserDeps(r io.Reader) (userDeps string, foundMarker bool, err error) {
+	b := &bytes.Buffer{}
+	scanner := bufio.NewScanner(r)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		b.WriteString(line)
+		b.WriteString("\n")
+		if strings.HasPrefix(line, depManifestKBMarker) {
+			foundMarker = true
+			userDeps = b.String()
+			return
+		}
+	}
+
+	err = scanner.Err()
+	return
+}
+
+func addKubeBuilderDeps(w io.Writer) error {
+	depTmplArgs := map[string]string{
+		"Version": version.GetVersion().KubeBuilderVersion,
+	}
+	t := template.Must(template.New("dep-manifest-template").Parse(depManifestOverride))
+	return t.Execute(w, depTmplArgs)
+}

cmd/kubebuilder/initproject/vendor_test.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package initproject
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestTryReadingUserDeps(t *testing.T) {
+	tests := []struct {
+		in          string
+		expKBMarker bool
+		expUserDeps string
+	}{
+		{
+			in: `
+ABC
+ABC
+jslsls
+sjsslsls
+			`,
+			expKBMarker: false,
+			expUserDeps: "",
+		},
+		{
+			in: `
+ABC
+ABC
+# DO NOT MODIFY BELOW THIS LINE.
+jslsls
+sjsslsls
+			`,
+			expKBMarker: true,
+			expUserDeps: `
+ABC
+ABC
+# DO NOT MODIFY BELOW THIS LINE.
+`,
+		},
+		{
+			in: `
+ABC
+ABC
+# DO NOT MODIFY BELOW THIS LINE.
+			`,
+			expKBMarker: true,
+			expUserDeps: `
+ABC
+ABC
+# DO NOT MODIFY BELOW THIS LINE.
+`,
+		},
+	}
+
+	for _, test := range tests {
+		r := bytes.NewReader([]byte(test.in))
+		userDeps, kbMarker, err := tryReadingUserDeps(r)
+		if err != nil {
+			t.Errorf("Reading UserDeps should succeed, but got an error: %v", err)
+		}
+		if test.expKBMarker != kbMarker {
+			t.Errorf("KB marker mismatch: exp: '%v' got: '%v'", test.expKBMarker, kbMarker)
+		}
+		if test.expUserDeps != userDeps {
+			t.Errorf("UserDeps don't match: exp: '%v' got: '%v'", test.expUserDeps, userDeps)
+		}
+
+	}
+}

cmd/kubebuilder/update/vendor.go

@@ -19,10 +19,11 @@ package update
 import (
 	"github.com/spf13/cobra"
 
-	"github.com/emicklei/go-restful/log"
 	"github.com/kubernetes-sigs/kubebuilder/cmd/kubebuilder/initproject"
 )
 
+var overwriteDepManifest bool
+
 var vendorCmd = &cobra.Command{
 	Use:   "vendor",
 	Short: "Update the vendor packages managed by kubebuilder.",
@@ -35,10 +36,14 @@ kubebuilder update vendor
 
 func AddUpdateVendorCmd(cmd *cobra.Command) {
 	cmd.AddCommand(vendorCmd)
+	vendorCmd.Flags().BoolVar(&overwriteDepManifest, "overwrite-dep-manifest", false, "if true, overwrites the dep manifest file (Gopkg.toml)")
 }
 
 func RunUpdateVendor(cmd *cobra.Command, args []string) {
 	initproject.Update = true
-	log.Printf("Replacing vendored libraries managed by kubebuilder with the current version.")
+	if overwriteDepManifest {
+		// suppress the update behavior
+		initproject.Update = false
+	}
 	initproject.RunVendorInstall(cmd, args)
 }

test.sh

@@ -489,9 +489,9 @@ function test_generated_controller {
   go test -v ./pkg/...
 }
 
-function run_dep_ensure {
- header_text "running dep ensure"
- dep ensure
+function test_vendor_update {
+ header_text "performing vendor update"
+ kubebuilder update vendor
 }
 
 function test_docs {
@@ -510,4 +510,7 @@ prepare_testdir_under_gopath
 generate_crd_resources
 test_docs
 test_generated_controller
+test_vendor_update
+# re-running controller tests post vendor update
+test_generated_controller
 exit $rc