Add --file-layer type to diff layers of images

Author: Priya Wadhwa

cmd/diff.go

@@ -31,14 +31,13 @@ import (
 )
 
 var filename string
-var layers bool
 
 var diffCmd = &cobra.Command{
 	Use:   "diff",
 	Short: "Compare two images: [image1] [image2]",
 	Long:  `Compares two images using the specifed analyzers as indicated via flags (see documentation for available ones).`,
 	Args: func(cmd *cobra.Command, args []string) error {
-		if err := validateArgs(args, checkDiffArgNum, checkIfValidAnalyzer, checkFilenameFlag, checkLayersFlag); err != nil {
+		if err := validateArgs(args, checkDiffArgNum, checkIfValidAnalyzer, checkFilenameFlag); err != nil {
 			return err
 		}
 		return nil
@@ -70,18 +69,6 @@ func checkFilenameFlag(_ []string) error {
 	return errors.New("Please include --types=file with the --filename flag")
 }
 
-func checkLayersFlag(_ []string) error {
-	if layers {
-		for _, t := range types {
-			if t == "file" {
-				return nil
-			}
-		}
-		return errors.New("Please include --types=file with the --layers flag")
-	}
-	return nil
-}
-
 func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
 	diffTypes, err := differs.GetAnalyzers(diffArgs)
 	if err != nil {
@@ -115,12 +102,6 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
 		defer pkgutil.CleanupImage(*imageMap[image2Arg])
 	}
 
-	if layers {
-		if err := diffLayers(imageMap[image1Arg], imageMap[image2Arg]); err != nil {
-			return err
-		}
-	}
-
 	output.PrintToStdErr("Computing diffs\n")
 	req := differs.DiffRequest{
 		Image1:    *imageMap[image1Arg],
@@ -158,7 +139,6 @@ func diffFile(image1, image2 *pkgutil.Image) error {
 
 func init() {
 	diffCmd.Flags().StringVarP(&filename, "filename", "f", "", "Set this flag to the path of a file in both containers to view the diff of the file. Must be used with --types=file flag.")
-	diffCmd.Flags().BoolVarP(&layers, "layers", "", false, "Set this flag to diff the filesystems of two images by layer. Must be used with --types=file flag.")
 	RootCmd.AddCommand(diffCmd)
 	addSharedFlags(diffCmd)
 	output.AddFlags(diffCmd)

cmd/root.go

@@ -167,22 +167,56 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 	// TODO(nkubala): implement caching
 
 	// create tempdir and extract fs into it
+	var layers []pkgutil.Layer
+	if includeLayers() {
+		imgLayers, err := img.Layers()
+		if err != nil {
+			return pkgutil.Image{}, err
+		}
+		for _, layer := range imgLayers {
+			path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
+			if err != nil {
+				return pkgutil.Image{
+					Layers: layers,
+				}, err
+			}
+			if err := pkgutil.GetFileSystemForLayer(layer, path, nil); err != nil {
+				return pkgutil.Image{
+					Layers: layers,
+				}, err
+			}
+			layers = append(layers, pkgutil.Layer{
+				FSPath: path,
+			})
+		}
+	}
 	path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
 	if err != nil {
 		return pkgutil.Image{}, err
 	}
 	if err := pkgutil.GetFileSystemForImage(img, path, nil); err != nil {
 		return pkgutil.Image{
 			FSPath: path,
+			Layers: layers,
 		}, err
 	}
 	return pkgutil.Image{
 		Image:  img,
 		Source: imageName,
 		FSPath: path,
+		Layers: layers,
 	}, nil
 }
 
+func includeLayers() bool {
+	for _, t := range types {
+		if t == "file-layer" {
+			return true
+		}
+	}
+	return false
+}
+
 func init() {
 	RootCmd.PersistentFlags().StringVarP(&LogLevel, "verbosity", "v", "warning", "This flag controls the verbosity of container-diff.")
 	RootCmd.PersistentFlags().StringVarP(&format, "format", "", "", "Format to output diff in.")

differs/differs.go

@@ -42,13 +42,14 @@ type Analyzer interface {
 }
 
 var Analyzers = map[string]Analyzer{
-	"history":  HistoryAnalyzer{},
-	"metadata": MetadataAnalyzer{},
-	"file":     FileAnalyzer{},
-	"apt":      AptAnalyzer{},
-	"rpm":      RPMAnalyzer{},
-	"pip":      PipAnalyzer{},
-	"node":     NodeAnalyzer{},
+	"history":    HistoryAnalyzer{},
+	"metadata":   MetadataAnalyzer{},
+	"file":       FileAnalyzer{},
+	"file-layer": FileLayerAnalyzer{},
+	"apt":        AptAnalyzer{},
+	"rpm":        RPMAnalyzer{},
+	"pip":        PipAnalyzer{},
+	"node":       NodeAnalyzer{},
 }
 
 func (req DiffRequest) GetDiff() (map[string]util.Result, error) {

differs/file_diff.go

@@ -19,6 +19,8 @@ package differs
 import (
 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
 	"github.com/GoogleContainerTools/container-diff/util"
+	"io/ioutil"
+	"os"
 )
 
 type FileAnalyzer struct {
@@ -30,7 +32,7 @@ func (a FileAnalyzer) Name() string {
 
 // FileDiff diffs two packages and compares their contents
 func (a FileAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) {
-	diff, err := diffImageFiles(image1, image2)
+	diff, err := diffImageFiles(image1.FSPath, image2.FSPath)
 	return &util.DirDiffResult{
 		Image1:   image1.Source,
 		Image2:   image2.Source,
@@ -53,10 +55,7 @@ func (a FileAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
 	return &result, err
 }
 
-func diffImageFiles(image1, image2 pkgutil.Image) (util.DirDiff, error) {
-	img1 := image1.FSPath
-	img2 := image2.FSPath
-
+func diffImageFiles(img1, img2 string) (util.DirDiff, error) {
 	var diff util.DirDiff
 
 	img1Dir, err := pkgutil.GetDirectory(img1, true)
@@ -71,3 +70,81 @@ func diffImageFiles(image1, image2 pkgutil.Image) (util.DirDiff, error) {
 	diff, _ = util.DiffDirectory(img1Dir, img2Dir)
 	return diff, nil
 }
+
+type FileLayerAnalyzer struct {
+}
+
+func (a FileLayerAnalyzer) Name() string {
+	return "FileLayerAnalyzer"
+}
+
+// FileDiff diffs two packages and compares their contents
+func (a FileLayerAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) {
+	var dirDiffs []util.DirDiff
+	// This path to an empty dir will be used for diffing in cases
+	// where one image has more layers than the other
+	emptyPath, err := ioutil.TempDir("", "")
+	if err != nil {
+		return &util.MultipleDirDiffResult{}, err
+	}
+	defer os.RemoveAll(emptyPath)
+
+	// Go through each layer of the first image...
+	for index, layer := range image1.Layers {
+		// ...if there is no corresponding layer in the second image, diff with the empty dir
+		if index >= len(image2.Layers) {
+			diff, err := diffImageFiles(layer.FSPath, emptyPath)
+			if err != nil {
+				return &util.MultipleDirDiffResult{}, err
+			}
+			dirDiffs = append(dirDiffs, diff)
+			continue
+		}
+		// ...else, diff as usual
+		layer2 := image2.Layers[index]
+		diff, err := diffImageFiles(layer.FSPath, layer2.FSPath)
+		if err != nil {
+			return &util.MultipleDirDiffResult{}, err
+		}
+		dirDiffs = append(dirDiffs, diff)
+	}
+
+	// check if there are any additional layers in image2...
+	if len(image2.Layers) > len(image1.Layers) {
+		// ... and diff any additional layers with the empty dir
+		for index := len(image1.Layers); index < len(image2.Layers); index++ {
+			layer2 := image2.Layers[index]
+			diff, err := diffImageFiles(emptyPath, layer2.FSPath)
+			if err != nil {
+				return &util.MultipleDirDiffResult{}, err
+			}
+			dirDiffs = append(dirDiffs, diff)
+		}
+	}
+	return &util.MultipleDirDiffResult{
+		Image1:   image1.Source,
+		Image2:   image2.Source,
+		DiffType: "FileLayer",
+		Diff: util.MultipleDirDiff{
+			DirDiffs: dirDiffs,
+		},
+	}, nil
+}
+
+func (a FileLayerAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
+	var directoryEntries [][]pkgutil.DirectoryEntry
+	for _, layer := range image.Layers {
+		layerDir, err := pkgutil.GetDirectory(layer.FSPath, true)
+		if err != nil {
+			return util.FileLayerAnalyzeResult{}, err
+		}
+		directoryEntry := pkgutil.GetDirectoryEntries(layerDir)
+		directoryEntries = append(directoryEntries, directoryEntry)
+	}
+
+	return &util.FileLayerAnalyzeResult{
+		Image:       image.Source,
+		AnalyzeType: "FileLayer",
+		Analysis:    directoryEntries,
+	}, nil
+}

pkg/util/image_utils.go

@@ -33,10 +33,15 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+type Layer struct {
+	FSPath string
+}
+
 type Image struct {
 	Image  v1.Image
 	Source string
 	FSPath string
+	Layers []Layer
 }
 
 type ImageHistoryItem struct {
@@ -50,6 +55,13 @@ func CleanupImage(image Image) {
 			logrus.Warn(err.Error())
 		}
 	}
+	if image.Layers != nil {
+		for _, layer := range image.Layers {
+			if err := os.RemoveAll(layer.FSPath); err != nil {
+				logrus.Warn(err.Error())
+			}
+		}
+	}
 }
 
 func SortMap(m map[string]struct{}) string {
@@ -61,6 +73,15 @@ func SortMap(m map[string]struct{}) string {
 	return strings.Join(pairs, " ")
 }
 
+// GetFileSystemForLayer unpacks a layer to local disk
+func GetFileSystemForLayer(layer v1.Layer, root string, whitelist []string) error {
+	contents, err := layer.Uncompressed()
+	if err != nil {
+		return err
+	}
+	return unpackTar(tar.NewReader(contents), root, whitelist)
+}
+
 // unpack image filesystem to local disk
 func GetFileSystemForImage(image v1.Image, root string, whitelist []string) error {
 	if err := unpackTar(tar.NewReader(mutate.Extract(image)), root, whitelist); err != nil {

util/analyze_output_utils.go

@@ -216,3 +216,55 @@ func (r FileAnalyzeResult) OutputText(analyzeType string, format string) error {
 	}
 	return TemplateOutputFromFormat(strResult, "FileAnalyze", format)
 }
+
+type FileLayerAnalyzeResult AnalyzeResult
+
+func (r FileLayerAnalyzeResult) OutputStruct() interface{} {
+	analysis, valid := r.Analysis.([][]util.DirectoryEntry)
+	if !valid {
+		logrus.Error("Unexpected structure of Analysis.  Should be of type []DirectoryEntry")
+		return errors.New("Could not output FileAnalyzer analysis result")
+	}
+
+	for _, a := range analysis {
+		if SortSize {
+			directoryBy(directorySizeSort).Sort(a)
+		} else {
+			directoryBy(directoryNameSort).Sort(a)
+		}
+	}
+
+	r.Analysis = analysis
+	return r
+}
+
+func (r FileLayerAnalyzeResult) OutputText(analyzeType string, format string) error {
+	analysis, valid := r.Analysis.([][]util.DirectoryEntry)
+	if !valid {
+		logrus.Error("Unexpected structure of Analysis.  Should be of type []DirectoryEntry")
+		return errors.New("Could not output FileAnalyzer analysis result")
+	}
+
+	var strDirectoryEntries [][]StrDirectoryEntry
+
+	for _, a := range analysis {
+		if SortSize {
+			directoryBy(directorySizeSort).Sort(a)
+		} else {
+			directoryBy(directoryNameSort).Sort(a)
+		}
+		strAnalysis := stringifyDirectoryEntries(a)
+		strDirectoryEntries = append(strDirectoryEntries, strAnalysis)
+	}
+
+	strResult := struct {
+		Image       string
+		AnalyzeType string
+		Analysis    [][]StrDirectoryEntry
+	}{
+		Image:       r.Image,
+		AnalyzeType: r.AnalyzeType,
+		Analysis:    strDirectoryEntries,
+	}
+	return TemplateOutputFromFormat(strResult, "FileLayerAnalyze", format)
+}