Merge pull request #233 from priyawadhwa/layers

Compare layers of images

Author: priyawadhwa

cmd/root.go

@@ -167,22 +167,56 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 	// TODO(nkubala): implement caching
 
 	// create tempdir and extract fs into it
+	var layers []pkgutil.Layer
+	if includeLayers() {
+		imgLayers, err := img.Layers()
+		if err != nil {
+			return pkgutil.Image{}, err
+		}
+		for _, layer := range imgLayers {
+			path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
+			if err != nil {
+				return pkgutil.Image{
+					Layers: layers,
+				}, err
+			}
+			if err := pkgutil.GetFileSystemForLayer(layer, path, nil); err != nil {
+				return pkgutil.Image{
+					Layers: layers,
+				}, err
+			}
+			layers = append(layers, pkgutil.Layer{
+				FSPath: path,
+			})
+		}
+	}
 	path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
 	if err != nil {
 		return pkgutil.Image{}, err
 	}
 	if err := pkgutil.GetFileSystemForImage(img, path, nil); err != nil {
 		return pkgutil.Image{
 			FSPath: path,
+			Layers: layers,
 		}, err
 	}
 	return pkgutil.Image{
 		Image:  img,
 		Source: imageName,
 		FSPath: path,
+		Layers: layers,
 	}, nil
 }
 
+func includeLayers() bool {
+	for _, t := range types {
+		if t == "layer" {
+			return true
+		}
+	}
+	return false
+}
+
 func init() {
 	RootCmd.PersistentFlags().StringVarP(&LogLevel, "verbosity", "v", "warning", "This flag controls the verbosity of container-diff.")
 	RootCmd.PersistentFlags().StringVarP(&format, "format", "", "", "Format to output diff in.")

differs/differs.go

@@ -45,6 +45,7 @@ var Analyzers = map[string]Analyzer{
 	"history":  HistoryAnalyzer{},
 	"metadata": MetadataAnalyzer{},
 	"file":     FileAnalyzer{},
+	"layer":    FileLayerAnalyzer{},
 	"apt":      AptAnalyzer{},
 	"rpm":      RPMAnalyzer{},
 	"pip":      PipAnalyzer{},

differs/file_diff.go

@@ -19,6 +19,7 @@ package differs
 import (
 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
 	"github.com/GoogleContainerTools/container-diff/util"
+	"github.com/sirupsen/logrus"
 )
 
 type FileAnalyzer struct {
@@ -30,7 +31,7 @@ func (a FileAnalyzer) Name() string {
 
 // FileDiff diffs two packages and compares their contents
 func (a FileAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) {
-	diff, err := diffImageFiles(image1, image2)
+	diff, err := diffImageFiles(image1.FSPath, image2.FSPath)
 	return &util.DirDiffResult{
 		Image1:   image1.Source,
 		Image2:   image2.Source,
@@ -53,10 +54,7 @@ func (a FileAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
 	return &result, err
 }
 
-func diffImageFiles(image1, image2 pkgutil.Image) (util.DirDiff, error) {
-	img1 := image1.FSPath
-	img2 := image2.FSPath
-
+func diffImageFiles(img1, img2 string) (util.DirDiff, error) {
 	var diff util.DirDiff
 
 	img1Dir, err := pkgutil.GetDirectory(img1, true)
@@ -71,3 +69,63 @@ func diffImageFiles(image1, image2 pkgutil.Image) (util.DirDiff, error) {
 	diff, _ = util.DiffDirectory(img1Dir, img2Dir)
 	return diff, nil
 }
+
+type FileLayerAnalyzer struct {
+}
+
+func (a FileLayerAnalyzer) Name() string {
+	return "FileLayerAnalyzer"
+}
+
+// FileDiff diffs two packages and compares their contents
+func (a FileLayerAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) {
+	var dirDiffs []util.DirDiff
+
+	// Go through each layer of the first image...
+	for index, layer := range image1.Layers {
+		if index >= len(image2.Layers) {
+			continue
+		}
+		// ...else, diff as usual
+		layer2 := image2.Layers[index]
+		diff, err := diffImageFiles(layer.FSPath, layer2.FSPath)
+		if err != nil {
+			return &util.MultipleDirDiffResult{}, err
+		}
+		dirDiffs = append(dirDiffs, diff)
+	}
+
+	// check if there are any additional layers in either image
+	if len(image1.Layers) != len(image2.Layers) {
+		if len(image1.Layers) > len(image2.Layers) {
+			logrus.Infof("%s has additional layers, please use container-diff analyze to view the files in these layers", image1.Source)
+		} else {
+			logrus.Infof("%s has additional layers, please use container-diff analyze to view the files in these layers", image2.Source)
+		}
+	}
+	return &util.MultipleDirDiffResult{
+		Image1:   image1.Source,
+		Image2:   image2.Source,
+		DiffType: "FileLayer",
+		Diff: util.MultipleDirDiff{
+			DirDiffs: dirDiffs,
+		},
+	}, nil
+}
+
+func (a FileLayerAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
+	var directoryEntries [][]pkgutil.DirectoryEntry
+	for _, layer := range image.Layers {
+		layerDir, err := pkgutil.GetDirectory(layer.FSPath, true)
+		if err != nil {
+			return util.FileLayerAnalyzeResult{}, err
+		}
+		directoryEntries = append(directoryEntries, pkgutil.GetDirectoryEntries(layerDir))
+	}
+
+	return &util.FileLayerAnalyzeResult{
+		Image:       image.Source,
+		AnalyzeType: "FileLayer",
+		Analysis:    directoryEntries,
+	}, nil
+}

pkg/util/image_utils.go

@@ -33,10 +33,15 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+type Layer struct {
+	FSPath string
+}
+
 type Image struct {
 	Image  v1.Image
 	Source string
 	FSPath string
+	Layers []Layer
 }
 
 type ImageHistoryItem struct {
@@ -50,6 +55,13 @@ func CleanupImage(image Image) {
 			logrus.Warn(err.Error())
 		}
 	}
+	if image.Layers != nil {
+		for _, layer := range image.Layers {
+			if err := os.RemoveAll(layer.FSPath); err != nil {
+				logrus.Warn(err.Error())
+			}
+		}
+	}
 }
 
 func SortMap(m map[string]struct{}) string {
@@ -61,6 +73,15 @@ func SortMap(m map[string]struct{}) string {
 	return strings.Join(pairs, " ")
 }
 
+// GetFileSystemForLayer unpacks a layer to local disk
+func GetFileSystemForLayer(layer v1.Layer, root string, whitelist []string) error {
+	contents, err := layer.Uncompressed()
+	if err != nil {
+		return err
+	}
+	return unpackTar(tar.NewReader(contents), root, whitelist)
+}
+
 // unpack image filesystem to local disk
 func GetFileSystemForImage(image v1.Image, root string, whitelist []string) error {
 	if err := unpackTar(tar.NewReader(mutate.Extract(image)), root, whitelist); err != nil {

tests/file_layer_analysis_expected.json

@@ -0,0 +1,26 @@
+[
+  {
+    "Image": "gcr.io/gcp-runtimes/diff-layer-base",
+    "AnalyzeType": "FileLayer",
+    "Analysis": [
+      [
+        {
+          "Name": "/first",
+          "Size": 6
+        }
+      ],
+      [
+        {
+          "Name": "/second",
+          "Size": 7
+        }
+      ],
+      [
+        {
+          "Name": "/third",
+          "Size": 6
+        }
+      ]
+    ]
+  }
+]

tests/file_layer_diff_expected.json

@@ -0,0 +1,31 @@
+[
+  {
+    "Image1": "gcr.io/gcp-runtimes/diff-layer-base",
+    "Image2": "gcr.io/gcp-runtimes/diff-layer-modified",
+    "DiffType": "FileLayer",
+    "Diff": {
+      "DirDiffs": [
+        {
+          "Adds": null,
+          "Dels": null,
+          "Mods": null
+        },
+        {
+          "Adds": [
+            {
+              "Name": "/modified",
+              "Size": 9
+            }
+          ],
+          "Dels": [
+            {
+              "Name": "/second",
+              "Size": 7
+            }
+          ],
+          "Mods": null
+        }
+      ]
+    }
+  }
+]

tests/integration_test.go

@@ -38,6 +38,9 @@ const (
 	diffBase     = "gcr.io/gcp-runtimes/diff-base"
 	diffModified = "gcr.io/gcp-runtimes/diff-modified"
 
+	diffLayerBase    = "gcr.io/gcp-runtimes/diff-layer-base"
+	diffLayerModifed = "gcr.io/gcp-runtimes/diff-layer-modified"
+
 	metadataBase     = "gcr.io/gcp-runtimes/metadata-base"
 	metadataModified = "gcr.io/gcp-runtimes/metadata-modified"
 
@@ -106,6 +109,14 @@ func TestDiffAndAnalysis(t *testing.T) {
 			differFlags:  []string{"--type=file"},
 			expectedFile: "file_diff_expected.json",
 		},
+		{
+			description:  "file layer differ",
+			subcommand:   "diff",
+			imageA:       diffLayerBase,
+			imageB:       diffLayerModifed,
+			differFlags:  []string{"--type=layer"},
+			expectedFile: "file_layer_diff_expected.json",
+		},
 		{
 			description:  "apt differ",
 			subcommand:   "diff",
@@ -191,6 +202,13 @@ func TestDiffAndAnalysis(t *testing.T) {
 			differFlags:  []string{"--type=file", "-o"},
 			expectedFile: "file_sorted_analysis_expected.json",
 		},
+		{
+			description:  "file layer analysis",
+			subcommand:   "analyze",
+			imageA:       diffLayerBase,
+			differFlags:  []string{"--type=layer"},
+			expectedFile: "file_layer_analysis_expected.json",
+		},
 		{
 			description:  "pip analysis",
 			subcommand:   "analyze",

util/analyze_output_utils.go

@@ -216,3 +216,55 @@ func (r FileAnalyzeResult) OutputText(analyzeType string, format string) error {
 	}
 	return TemplateOutputFromFormat(strResult, "FileAnalyze", format)
 }
+
+type FileLayerAnalyzeResult AnalyzeResult
+
+func (r FileLayerAnalyzeResult) OutputStruct() interface{} {
+	analysis, valid := r.Analysis.([][]util.DirectoryEntry)
+	if !valid {
+		logrus.Error("Unexpected structure of Analysis.  Should be of type []DirectoryEntry")
+		return errors.New("Could not output FileAnalyzer analysis result")
+	}
+
+	for _, a := range analysis {
+		if SortSize {
+			directoryBy(directorySizeSort).Sort(a)
+		} else {
+			directoryBy(directoryNameSort).Sort(a)
+		}
+	}
+
+	r.Analysis = analysis
+	return r
+}
+
+func (r FileLayerAnalyzeResult) OutputText(analyzeType string, format string) error {
+	analysis, valid := r.Analysis.([][]util.DirectoryEntry)
+	if !valid {
+		logrus.Error("Unexpected structure of Analysis.  Should be of type []DirectoryEntry")
+		return errors.New("Could not output FileAnalyzer analysis result")
+	}
+
+	var strDirectoryEntries [][]StrDirectoryEntry
+
+	for _, a := range analysis {
+		if SortSize {
+			directoryBy(directorySizeSort).Sort(a)
+		} else {
+			directoryBy(directoryNameSort).Sort(a)
+		}
+		strAnalysis := stringifyDirectoryEntries(a)
+		strDirectoryEntries = append(strDirectoryEntries, strAnalysis)
+	}
+
+	strResult := struct {
+		Image       string
+		AnalyzeType string
+		Analysis    [][]StrDirectoryEntry
+	}{
+		Image:       r.Image,
+		AnalyzeType: r.AnalyzeType,
+		Analysis:    strDirectoryEntries,
+	}
+	return TemplateOutputFromFormat(strResult, "FileLayerAnalyze", format)
+}