Merge pull request #107 from dlorenc/mem

Unpack layers in memory instead of extracting to disk first.

Author: dlorenc

pkg/util/docker_utils.go

@@ -17,13 +17,15 @@ limitations under the License.
 package util
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
-	"path/filepath"
+	"strings"
 
 	"github.com/docker/docker/client"
 	"github.com/golang/glog"
@@ -49,56 +51,73 @@ func NewClient() (*client.Client, error) {
 	return cli, nil
 }
 
-func getLayersFromManifest(manifestPath string) ([]string, error) {
+func getLayersFromManifest(r io.Reader) ([]string, error) {
 	type Manifest struct {
 		Layers []string
 	}
 
-	manifestJSON, err := ioutil.ReadFile(manifestPath)
+	manifestJSON, err := ioutil.ReadAll(r)
 	if err != nil {
-		errMsg := fmt.Sprintf("Could not open manifest to get layer order: %s", err)
-		return []string{}, errors.New(errMsg)
+		return nil, err
 	}
 
 	var imageManifest []Manifest
-	err = json.Unmarshal(manifestJSON, &imageManifest)
-	if err != nil {
-		errMsg := fmt.Sprintf("Could not unmarshal manifest to get layer order: %s", err)
-		return []string{}, errors.New(errMsg)
+	if err := json.Unmarshal(manifestJSON, &imageManifest); err != nil {
+		return []string{}, fmt.Errorf("Could not unmarshal manifest to get layer order: %s", err)
 	}
 	return imageManifest[0].Layers, nil
 }
 
 func unpackDockerSave(tarPath string, target string) error {
 	if _, ok := os.Stat(target); ok != nil {
-		os.MkdirAll(target, 0777)
+		os.MkdirAll(target, 0775)
 	}
-
-	tempLayerDir, err := ioutil.TempDir("", ".container-diff")
+	f, err := os.Open(tarPath)
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(tempLayerDir)
 
-	if err := UnTar(tarPath, tempLayerDir); err != nil {
-		errMsg := fmt.Sprintf("Could not unpack saved Docker image %s: %s", tarPath, err)
-		return errors.New(errMsg)
-	}
+	tr := tar.NewReader(f)
 
-	manifest := filepath.Join(tempLayerDir, "manifest.json")
-	layers, err := getLayersFromManifest(manifest)
-	if err != nil {
-		return err
-	}
+	// Unpack the layers into a map, since we need to sort out the order later.
+	var layers []string
+	layerMap := map[string][]byte{}
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
 
-	for _, layer := range layers {
-		layerTar := filepath.Join(tempLayerDir, layer)
-		if _, err := os.Stat(layerTar); err != nil {
-			glog.Infof("Did not unpack layer %s because no layer.tar found", layer)
+		// Docker save contains files and directories. Ignore the directories.
+		// We care about the layers and the manifest. The layers look like:
+		// $SHA/layer.tar
+		// and they are referenced that way in the manifest.
+		switch t := hdr.Typeflag; t {
+		case tar.TypeReg:
+			if hdr.Name == "manifest.json" {
+				layers, err = getLayersFromManifest(tr)
+				if err != nil {
+					return err
+				}
+			} else if strings.HasSuffix(hdr.Name, ".tar") {
+				layerMap[hdr.Name], err = ioutil.ReadAll(tr)
+				if err != nil {
+					return err
+				}
+			}
+		case tar.TypeDir:
 			continue
+		default:
+			return fmt.Errorf("unsupported file type %v found in file %s tar %s", t, hdr.Name, tarPath)
 		}
-		if err = UnTar(layerTar, target); err != nil {
-			glog.Errorf("Could not unpack layer %s: %s", layer, err)
+	}
+
+	for _, layer := range layers {
+		if err = UnTar(bytes.NewReader(layerMap[layer]), target); err != nil {
+			return fmt.Errorf("Could not unpack layer %s: %s", layer, err)
 		}
 	}
 	return nil

pkg/util/tar_prepper.go

@@ -55,7 +55,12 @@ func (p TarPrepper) GetConfig() (ConfigSchema, error) {
 		return ConfigSchema{}, nil
 	}
 	defer os.RemoveAll(tempDir)
-	if err := UnTar(p.Source, tempDir); err != nil {
+	f, err := os.Open(p.Source)
+	if err != nil {
+		return ConfigSchema{}, err
+	}
+	defer f.Close()
+	if err := UnTar(f, tempDir); err != nil {
 		return ConfigSchema{}, err
 	}
 

pkg/util/tar_utils.go

@@ -95,19 +95,13 @@ func unpackTar(tr *tar.Reader, path string) error {
 
 // UnTar takes in a path to a tar file and writes the untarred version to the provided target.
 // Only untars one level, does not untar nested tars.
-func UnTar(filename string, target string) error {
+func UnTar(r io.Reader, target string) error {
 	if _, ok := os.Stat(target); ok != nil {
-		os.MkdirAll(target, 0777)
+		os.MkdirAll(target, 0775)
 	}
-	file, err := os.Open(filename)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-	tr := tar.NewReader(file)
-	err = unpackTar(tr, target)
-	if err != nil {
-		glog.Error(err)
+
+	tr := tar.NewReader(r)
+	if err := unpackTar(tr, target); err != nil {
 		return err
 	}
 	return nil

util/tar_utils_test.go

@@ -82,8 +82,11 @@ func TestUnTar(t *testing.T) {
 		if test.starter != "" {
 			CopyDir(test.starter, test.target)
 		}
-		err := pkgutil.UnTar(test.tarPath, test.target)
-		if err != nil && !test.err {
+		r, err := os.Open(test.tarPath)
+		if err != nil {
+			t.Errorf("Error opening tar: %s", err)
+		}
+		if err := pkgutil.UnTar(r, test.target); err != nil && !test.err {
 			t.Errorf(test.descrip, "Got unexpected error: %s", err)
 			remove = false
 		}