Merge pull request #57 from r2d4/integration-tests

Use go integration tests

Author: r2d4

.container-diff-tests.sh

@@ -1,2 +1 @@
-tests/*_actual.json
 out/*

.gitignore

@@ -1,11 +1,8 @@
-sudo: required
-dist: trusty
-
 language: go
+os: linux
 go:
-        - 1.8.1
+        - 1.8.3
 go_import_path: github.com/GoogleCloudPlatform/container-diff
 
 script:
- - ./.gofmt.sh
- - travis_wait ./.container-diff-tests.sh
+ - make test integration

.travis.yml

@@ -9,7 +9,7 @@ ORG := github.com/GoogleCloudPlatform
 PROJECT := container-diff
 REPOPATH ?= $(ORG)/$(PROJECT)
 
-SUPPORTED_PLATFORMS := linux-amd64 darwin-amd64
+SUPPORTED_PLATFORMS := linux-amd64 darwin-amd64 windows-amd64
 BUILD_PACKAGE = $(REPOPATH)
 
 # These build tags are from the containers/image library.
@@ -33,7 +33,11 @@ cross: $(foreach platform, $(SUPPORTED_PLATFORMS), out/$(PROJECT)-$(platform))
 
 .PHONY: test
 test: $(BUILD_DIR)/$(PROJECT)
-	./.container-diff-tests.sh
+	@ ./test.sh
+
+.PHONY: integration
+integration: $(BUILD_DIR)/$(PROJECT)
+	go test -v -tags integration $(REPOPATH)/tests
 
 .PHONY: clean
 clean:

Makefile

@@ -16,6 +16,10 @@
 
 set -e
 
+echo "Running go tests..."
+go test `go list ./... | grep -v vendor`
+
+echo "Checking gofmt..."
 files=$(find . -name "*.go" | grep -v vendor/ | xargs gofmt -l -s)
 if [[ $files ]]; then
     echo "Gofmt errors in files: $files"

.gofmt.sh renamed to test.sh

@@ -0,0 +1,170 @@
+// +build integration
+
+package tests
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+const (
+	diffBase     = "gcr.io/gcp-runtimes/diff-base"
+	diffModified = "gcr.io/gcp-runtimes/diff-modified"
+
+	aptBase     = "gcr.io/gcp-runtimes/apt-base"
+	aptModified = "gcr.io/gcp-runtimes/apt-modified"
+
+	// Why is this node-modified:2.0?
+	nodeBase     = "gcr.io/gcp-runtimes/node-modified:2.0"
+	nodeModified = "gcr.io/gcp-runtimes/node-modified"
+
+	pipModified = "gcr.io/gcp-runtimes/pip-modified"
+
+	multiBase     = "gcr.io/gcp-runtimes/multi-base"
+	multiModified = "gcr.io/gcp-runtimes/multi-modified"
+)
+
+type ContainerDiffRunner struct {
+	t          *testing.T
+	binaryPath string
+}
+
+func (c *ContainerDiffRunner) Run(command ...string) (string, error) {
+	path, err := filepath.Abs(c.binaryPath)
+	if err != nil {
+		c.t.Fatalf("Error finding container-diff binary: %s", err)
+	}
+	c.t.Logf("Running command: %s %s", path, command)
+	cmd := exec.Command(path, command...)
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("Error running command %s: %s Stderr: %s", command, err, stderr.String())
+	}
+	return stdout.String(), nil
+}
+
+func TestDiffAndAnalysis(t *testing.T) {
+	runner := ContainerDiffRunner{
+		t:          t,
+		binaryPath: "../out/container-diff",
+	}
+
+	var tests = []struct {
+		description string
+		imageA      string
+		imageB      string
+		differFlag  string
+		subcommand  string
+
+		//TODO: Don't consume a json file
+		expectedFile string
+	}{
+		{
+			description:  "file differ",
+			subcommand:   "diff",
+			imageA:       diffBase,
+			imageB:       diffModified,
+			differFlag:   "-f",
+			expectedFile: "file_diff_expected.json",
+		},
+		{
+			description:  "apt differ",
+			subcommand:   "diff",
+			imageA:       aptBase,
+			imageB:       aptModified,
+			differFlag:   "-a",
+			expectedFile: "apt_diff_expected.json",
+		},
+		{
+			description:  "node differ",
+			subcommand:   "diff",
+			imageA:       nodeBase,
+			imageB:       nodeModified,
+			differFlag:   "-n",
+			expectedFile: "node_diff_order_expected.json",
+		},
+		{
+			description:  "multi differ",
+			subcommand:   "diff",
+			imageA:       multiBase,
+			imageB:       multiModified,
+			differFlag:   "-npa",
+			expectedFile: "multi_diff_expected.json",
+		},
+		{
+			description:  "history differ",
+			subcommand:   "diff",
+			imageA:       diffBase,
+			imageB:       diffModified,
+			differFlag:   "-d",
+			expectedFile: "hist_diff_expected.json",
+		},
+		{
+			description:  "apt sorted differ",
+			subcommand:   "diff",
+			imageA:       aptBase,
+			imageB:       aptModified,
+			differFlag:   "-ao",
+			expectedFile: "apt_sorted_diff_expected.json",
+		},
+		{
+			description:  "apt analysis",
+			subcommand:   "analyze",
+			imageA:       aptModified,
+			differFlag:   "-a",
+			expectedFile: "apt_analysis_expected.json",
+		},
+		{
+			description:  "file sorted analysis",
+			subcommand:   "analyze",
+			imageA:       diffModified,
+			differFlag:   "-fo",
+			expectedFile: "file_sorted_analysis_expected.json",
+		},
+		{
+			description:  "pip analysis",
+			subcommand:   "analyze",
+			imageA:       pipModified,
+			differFlag:   "-p",
+			expectedFile: "pip_analysis_expected.json",
+		},
+		{
+			description:  "node analysis",
+			subcommand:   "analyze",
+			imageA:       nodeModified,
+			differFlag:   "-n",
+			expectedFile: "node_analysis_expected.json",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			args := []string{test.subcommand, test.imageA}
+			if test.imageB != "" {
+				args = append(args, test.imageB)
+			}
+			args = append(args, test.differFlag)
+			args = append(args, "-j")
+			actual, err := runner.Run(args...)
+			if err != nil {
+				t.Fatalf("Error running command: %s", err)
+			}
+			e, err := ioutil.ReadFile(test.expectedFile)
+			if err != nil {
+				t.Fatalf("Error reading expected file output file: %s", err)
+			}
+			actual = strings.TrimSpace(actual)
+			expected := strings.TrimSpace(string(e))
+			if actual != expected {
+				t.Errorf("Error actual output does not match expected.  \n\nExpected: %s\n\n Actual: %s", expected, actual)
+			}
+		})
+	}
+}