WIP: adds a simple layer cache.

Author: dlorenc

cmd/diff.go

@@ -24,6 +24,7 @@ import (
 	"sync"
 
 	"github.com/GoogleCloudPlatform/container-diff/differs"
+	"github.com/GoogleCloudPlatform/container-diff/pkg/cache"
 	pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util"
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
@@ -68,6 +69,12 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
 		return err
 	}
 	defer cli.Close()
+
+	fsCache, err := cache.NewFileCache(cacheDir())
+	if err != nil {
+		return err
+	}
+
 	var wg sync.WaitGroup
 	wg.Add(2)
 
@@ -83,6 +90,7 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
 			ip := pkgutil.ImagePrepper{
 				Source: imageName,
 				Client: cli,
+				Cache:  fsCache,
 			}
 			image, err := ip.GetImage()
 			imageMap[imageName] = &image

cmd/root.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	goflag "flag"
 	"fmt"
+	"os/user"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -37,6 +39,15 @@ var types string
 
 type validatefxn func(args []string) error
 
+func cacheDir() string {
+	user, err := user.Current()
+	if err != nil {
+		glog.Exit(err)
+	}
+	rootDir := filepath.Join(user.HomeDir, ".container-diff")
+	return filepath.Join(rootDir, "cache")
+}
+
 var RootCmd = &cobra.Command{
 	Use:   "container-diff",
 	Short: "container-diff is a tool for analyzing and comparing container images",

pkg/cache/cache.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2017 Google, Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+)
+
+type FileCache struct {
+	RootDir string
+}
+
+func NewFileCache(rootDir string) (*FileCache, error) {
+	if err := os.MkdirAll(rootDir, 0700); err != nil {
+		return nil, err
+	}
+	return &FileCache{RootDir: rootDir}, nil
+}
+
+func (f *FileCache) HasLayer(layerId string) bool {
+	_, err := os.Stat(filepath.Join(f.RootDir, layerId))
+	return !os.IsNotExist(err)
+}
+
+func (f *FileCache) SetLayer(layerId string, r io.Reader) (io.ReadCloser, error) {
+	path := filepath.Join(f.RootDir, layerId)
+	l, err := os.Create(path)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := io.Copy(l, r); err != nil {
+		return nil, err
+	}
+
+	return f.GetLayer(layerId)
+}
+
+func (f *FileCache) GetLayer(layerId string) (io.ReadCloser, error) {
+	path := filepath.Join(f.RootDir, layerId)
+	return os.Open(path)
+}

pkg/util/cloud_prepper.go

@@ -50,7 +50,7 @@ func (p CloudPrepper) GetFileSystem() (string, error) {
 		return "", err
 	}
 
-	return getFileSystemFromReference(ref, p.Source)
+	return getFileSystemFromReference(ref, p.Source, p.Cache)
 }
 
 func (p CloudPrepper) GetConfig() (ConfigSchema, error) {

pkg/util/daemon_prepper.go

@@ -49,7 +49,7 @@ func (p DaemonPrepper) GetFileSystem() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return getFileSystemFromReference(ref, p.Source)
+	return getFileSystemFromReference(ref, p.Source, p.Cache)
 }
 
 func (p DaemonPrepper) GetConfig() (ConfigSchema, error) {

pkg/util/image_prep_utils.go

@@ -20,11 +20,13 @@ import (
 	"archive/tar"
 	"encoding/json"
 	"errors"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/GoogleCloudPlatform/container-diff/pkg/cache"
 	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/types"
 	"github.com/golang/glog"
@@ -62,7 +64,7 @@ func getImageFromTar(tarPath string) (string, error) {
 	return path, err
 }
 
-func getFileSystemFromReference(ref types.ImageReference, imageName string) (string, error) {
+func getFileSystemFromReference(ref types.ImageReference, imageName string, c *cache.FileCache) (string, error) {
 	sanitizedName := strings.Replace(imageName, ":", "", -1)
 	sanitizedName = strings.Replace(sanitizedName, "/", "", -1)
 
@@ -85,7 +87,18 @@ func getFileSystemFromReference(ref types.ImageReference, imageName string) (str
 	}
 
 	for _, b := range img.LayerInfos() {
-		bi, _, err := imgSrc.GetBlob(b)
+		var bi io.ReadCloser
+		var err error
+		if c.HasLayer(b.Digest.String()) {
+			bi, err = c.GetLayer(b.Digest.String())
+		} else {
+			bi, _, err = imgSrc.GetBlob(b)
+			if err != nil {
+				return "", err
+			}
+			// We need to get a new reader after caching the old one.
+			bi, err = c.SetLayer(b.Digest.String(), bi)
+		}
 		if err != nil {
 			glog.Errorf("Failed to pull image layer: %s", err)
 			return "", err

pkg/util/image_prepper.go

@@ -19,13 +19,15 @@ package util
 import (
 	"errors"
 
+	"github.com/GoogleCloudPlatform/container-diff/pkg/cache"
 	"github.com/docker/docker/client"
 	"github.com/golang/glog"
 )
 
 type ImagePrepper struct {
 	Source string
 	Client *client.Client
+	Cache  *cache.FileCache
 }
 
 type Prepper interface {