Skip to content

Commit d6f22e3

Browse files
AlexNPavelAlexNPavel
committed
*: add tests for incluster test image and fix bugs
This adds tests for the new incluster image testing mode as well as fixes some bugs that this test exposed
1 parent 49fc1c1 commit d6f22e3

File tree

6 files changed

+243
-25
lines changed

6 files changed

+243
-25
lines changed

‎commands/operator-sdk/cmd/test/cluster.go

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
var (
3131
testNamespace string
3232
kubeconfigCluster string
33+
imagePullPolicy bool
3334
)
3435

3536
func NewTestClusterCmd() *cobra.Command {
@@ -45,6 +46,7 @@ func NewTestClusterCmd() *cobra.Command {
4546
}
4647
testCmd.Flags().StringVarP(&testNamespace, "namespace", "n", "default", "Namespace to run tests in")
4748
testCmd.Flags().StringVarP(&kubeconfigCluster, "kubeconfig", "k", defaultKubeConfig, "Kubeconfig path")
49+
testCmd.Flags().BoolVarP(&imagePullPolicy, "imagePullPolicy", "i", false, "Set test pod image pull policy to 'Never'")
4850

4951
return testCmd
5052
}
@@ -53,6 +55,12 @@ func testClusterFunc(cmd *cobra.Command, args []string) error {
5355
if len(args) != 1 {
5456
return fmt.Errorf("operator-sdk test cluster requires exactly 1 argument")
5557
}
58+
var pullPolicy v1.PullPolicy
59+
if imagePullPolicy {
60+
pullPolicy = v1.PullNever
61+
} else {
62+
pullPolicy = v1.PullAlways
63+
}
5664
testPod := &v1.Pod{
5765
ObjectMeta: metav1.ObjectMeta{
5866
Name: "operator-test",
@@ -62,7 +70,7 @@ func testClusterFunc(cmd *cobra.Command, args []string) error {
6270
Containers: []v1.Container{{
6371
Name: "operator-test",
6472
Image: args[0],
65-
ImagePullPolicy: v1.PullAlways,
73+
ImagePullPolicy: pullPolicy,
6674
Command: []string{"/go-test.sh"},
6775
Env: []v1.EnvVar{{
6876
Name: "TEST_NAMESPACE",
@@ -83,12 +91,14 @@ func testClusterFunc(cmd *cobra.Command, args []string) error {
8391
if err != nil {
8492
return fmt.Errorf("failed to create test pod: %v", err)
8593
}
86-
defer func() {
87-
err = kubeclient.CoreV1().Pods(testNamespace).Delete(testPod.Name, &metav1.DeleteOptions{})
88-
if err != nil {
89-
fmt.Printf("Warning: failed to delete test pod")
90-
}
91-
}()
94+
/*
95+
defer func() {
96+
err = kubeclient.CoreV1().Pods(testNamespace).Delete(testPod.Name, &metav1.DeleteOptions{})
97+
if err != nil {
98+
fmt.Printf("Warning: failed to delete test pod")
99+
}
100+
}()
101+
*/
92102
for {
93103
testPod, err = kubeclient.CoreV1().Pods(testNamespace).Get(testPod.Name, metav1.GetOptions{})
94104
if err != nil {

‎pkg/generator/generator.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ const (
5454
build = "build.sh"
5555
dockerBuild = "docker_build.sh"
5656
dockerfile = "Dockerfile"
57-
testingDockerfile = "Dockerfile_Testing"
57+
testingDockerfile = "Dockerfile"
5858
goTest = "go-test.sh"
5959
boilerplate = "boilerplate.go.txt"
6060
updateGenerated = "update-generated.sh"
@@ -364,7 +364,11 @@ func renderBuildFiles(buildDir, repoPath, projectName string) error {
364364
return err
365365
}
366366

367-
return renderWriteFile(filepath.Join(buildDir, goTest), "tmp/build/go-test.sh", goTestScript, tmplData{})
367+
buf = &bytes.Buffer{}
368+
if err := renderFile(buf, filepath.Join(buildDir, goTest), goTestScript, tmplData{}); err != nil {
369+
return err
370+
}
371+
return writeFileAndPrint(filepath.Join(buildDir, goTest), buf.Bytes(), os.FileMode(int(0755)))
368372
}
369373

370374
func renderDockerBuildFile(w io.Writer) error {

‎test/e2e/incluster-test-code/main_test.go.tmpl

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright 2018 The Operator-SDK Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package e2e
16+
17+
import (
18+
"testing"
19+
20+
f "github.com/operator-framework/operator-sdk/pkg/test"
21+
)
22+
23+
func TestMain(m *testing.M) {
24+
f.MainEntry(m)
25+
}

‎test/e2e/incluster-test-code/memcached_test.go.tmpl

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
// Copyright 2018 The Operator-SDK Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package e2e
16+
17+
import (
18+
goctx "context"
19+
"fmt"
20+
"testing"
21+
"time"
22+
23+
operator "github.com/example-inc/memcached-operator/pkg/apis/cache/v1alpha1"
24+
framework "github.com/operator-framework/operator-sdk/pkg/test"
25+
"github.com/operator-framework/operator-sdk/pkg/test/e2eutil"
26+
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/apimachinery/pkg/types"
29+
)
30+
31+
var (
32+
retryInterval = time.Second * 5
33+
timeout = time.Second * 30
34+
)
35+
36+
func TestMemcached(t *testing.T) {
37+
memcachedList := &operator.MemcachedList{
38+
TypeMeta: metav1.TypeMeta{
39+
Kind: "Memcached",
40+
APIVersion: "cache.example.com/v1alpha1",
41+
},
42+
}
43+
err := framework.AddToFrameworkScheme(operator.AddToScheme, memcachedList)
44+
if err != nil {
45+
t.Fatalf("failed to add custom resource scheme to framework: %v", err)
46+
}
47+
// run subtests
48+
t.Run("memcached-group", func(t *testing.T) {
49+
t.Run("Cluster", MemcachedCluster)
50+
t.Run("Cluster2", MemcachedCluster)
51+
})
52+
}
53+
54+
func memcachedScaleTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error {
55+
namespace, err := ctx.GetNamespace()
56+
if err != nil {
57+
return fmt.Errorf("could not get namespace: %v", err)
58+
}
59+
// create memcached custom resource
60+
exampleMemcached := &operator.Memcached{
61+
TypeMeta: metav1.TypeMeta{
62+
Kind: "Memcached",
63+
APIVersion: "cache.example.com/v1alpha1",
64+
},
65+
ObjectMeta: metav1.ObjectMeta{
66+
Name: "example-memcached",
67+
Namespace: namespace,
68+
},
69+
Spec: operator.MemcachedSpec{
70+
Size: 3,
71+
},
72+
}
73+
err = f.DynamicClient.Create(goctx.TODO(), exampleMemcached)
74+
if err != nil {
75+
return err
76+
}
77+
ctx.AddFinalizerFn(func() error {
78+
return f.DynamicClient.Delete(goctx.TODO(), exampleMemcached)
79+
})
80+
// wait for example-memcached to reach 3 replicas
81+
err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-memcached", 3, retryInterval, timeout)
82+
if err != nil {
83+
return err
84+
}
85+
86+
err = f.DynamicClient.Get(goctx.TODO(), types.NamespacedName{Name: "example-memcached", Namespace: namespace}, exampleMemcached)
87+
if err != nil {
88+
return err
89+
}
90+
exampleMemcached.Spec.Size = 4
91+
err = f.DynamicClient.Update(goctx.TODO(), exampleMemcached)
92+
if err != nil {
93+
return err
94+
}
95+
96+
// wait for example-memcached to reach 4 replicas
97+
return e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-memcached", 4, retryInterval, timeout)
98+
}
99+
100+
func MemcachedCluster(t *testing.T) {
101+
t.Parallel()
102+
ctx := framework.NewTestCtx(t)
103+
defer ctx.Cleanup(t)
104+
err := ctx.InitializeClusterResources()
105+
if err != nil {
106+
t.Fatalf("failed to initialize cluster resources: %v", err)
107+
}
108+
t.Log("Initialized cluster resources")
109+
namespace, err := ctx.GetNamespace()
110+
if err != nil {
111+
t.Fatal(err)
112+
}
113+
// get global framework variables
114+
f := framework.Global
115+
// wait for memcached-operator to be ready
116+
err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, retryInterval, timeout)
117+
if err != nil {
118+
t.Fatal(err)
119+
}
120+
121+
if err = memcachedScaleTest(t, f, ctx); err != nil {
122+
t.Fatal(err)
123+
}
124+
}

‎test/e2e/memcached_test.go

Lines changed: 68 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,6 @@ func TestMemcached(t *testing.T) {
6161
ctx.AddFinalizerFn(func() error { return os.RemoveAll(path.Join(gopath, "/src/github.com/example-inc/memcached-operator")) })
6262

6363
os.Chdir("memcached-operator")
64-
os.RemoveAll("vendor/github.com/operator-framework/operator-sdk/pkg")
65-
os.Symlink(path.Join(gopath, "/src/github.com/operator-framework/operator-sdk/pkg"),
66-
"vendor/github.com/operator-framework/operator-sdk/pkg")
6764
handlerFile, err := os.Create("pkg/stub/handler.go")
6865
if err != nil {
6966
t.Fatal(err)
@@ -98,6 +95,33 @@ func TestMemcached(t *testing.T) {
9895
t.Fatalf("error: %v\nCommand Output: %s\n", err, string(cmdOut))
9996
}
10097

98+
t.Log("Copying test files to ./test")
99+
if err = os.MkdirAll("./test", os.FileMode(int(0755))); err != nil {
100+
t.Fatalf("could not create test/e2e dir: %v", err)
101+
}
102+
cmdOut, err = exec.Command("cp", "-a", path.Join(gopath, "/src/github.com/operator-framework/operator-sdk/test/e2e/incluster-test-code"), "./test/e2e").CombinedOutput()
103+
if err != nil {
104+
t.Fatalf("could not copy tests to test/e2e: %v\nCommand Output:\n%v", err, string(cmdOut))
105+
}
106+
// fix naming of files
107+
cmdOut, err = exec.Command("mv", "test/e2e/main_test.go.tmpl", "test/e2e/main_test.go").CombinedOutput()
108+
if err != nil {
109+
t.Fatalf("could not rename test/e2e/main_test.go.tmpl: %v\nCommand Output:\n%v", err, string(cmdOut))
110+
}
111+
cmdOut, err = exec.Command("mv", "test/e2e/memcached_test.go.tmpl", "test/e2e/memcached_test.go").CombinedOutput()
112+
if err != nil {
113+
t.Fatalf("could not rename test/e2e/memcached_test.go.tmpl: %v\nCommand Output:\n%v", err, string(cmdOut))
114+
}
115+
t.Log("Pulling new dependencies with dep ensure")
116+
cmdOut, err = exec.Command("dep", "ensure").CombinedOutput()
117+
if err != nil {
118+
t.Fatalf("dep ensure failed: %v\nCommand Output:\n%v", err, string(cmdOut))
119+
}
120+
// use current operator-sdk code
121+
os.RemoveAll("vendor/github.com/operator-framework/operator-sdk/pkg")
122+
os.Symlink(path.Join(gopath, "/src/github.com/operator-framework/operator-sdk/pkg"),
123+
"vendor/github.com/operator-framework/operator-sdk/pkg")
124+
101125
// create crd
102126
crdYAML, err := ioutil.ReadFile("deploy/crd.yaml")
103127
err = ctx.CreateFromYAML(crdYAML)
@@ -106,9 +130,10 @@ func TestMemcached(t *testing.T) {
106130
}
107131
t.Log("Created crd")
108132

109-
// run both subtests
133+
// run subtests
110134
t.Run("memcached-group", func(t *testing.T) {
111135
t.Run("Cluster", MemcachedCluster)
136+
t.Run("ClusterTest", MemcachedClusterTest)
112137
t.Run("Local", MemcachedLocal)
113138
})
114139
}
@@ -209,20 +234,10 @@ func MemcachedCluster(t *testing.T) {
209234
f := framework.Global
210235
ctx := f.NewTestCtx(t)
211236
defer ctx.Cleanup(t)
237+
operatorYAML, err := ioutil.ReadFile("deploy/operator.yaml")
212238
local := *f.ImageName == ""
213239
if local {
214240
*f.ImageName = "quay.io/example/memcached-operator:v0.0.1"
215-
}
216-
t.Log("Building operator docker image")
217-
cmdOut, err := exec.Command("operator-sdk", "build", *f.ImageName).CombinedOutput()
218-
if err != nil {
219-
t.Fatalf("error: %v\nCommand Output: %s\n", err, string(cmdOut))
220-
}
221-
222-
operatorYAML, err := ioutil.ReadFile("deploy/operator.yaml")
223-
operatorYAML = bytes.Replace(operatorYAML, []byte("REPLACE_IMAGE"), []byte(*f.ImageName), 1)
224-
225-
if local {
226241
if err != nil {
227242
t.Fatal(err)
228243
}
@@ -231,7 +246,19 @@ func MemcachedCluster(t *testing.T) {
231246
if err != nil {
232247
t.Fatal(err)
233248
}
234-
} else {
249+
}
250+
operatorYAML = bytes.Replace(operatorYAML, []byte("REPLACE_IMAGE"), []byte(*f.ImageName), 1)
251+
err = ioutil.WriteFile("deploy/operator.yaml", operatorYAML, os.FileMode(0644))
252+
if err != nil {
253+
t.Fatalf("failed to write deploy/operator.yaml: %v", err)
254+
}
255+
t.Log("Building operator docker image")
256+
cmdOut, err := exec.Command("operator-sdk", "build", *f.ImageName, "-e", "-t", "./test/e2e", "-n", "deploy/operator.yaml").CombinedOutput()
257+
if err != nil {
258+
t.Fatalf("error: %v\nCommand Output: %s\n", err, string(cmdOut))
259+
}
260+
261+
if !local {
235262
t.Log("Pushing docker image to repo")
236263
cmdOut, err = exec.Command("docker", "push", *f.ImageName).CombinedOutput()
237264
if err != nil {
@@ -275,3 +302,28 @@ func MemcachedCluster(t *testing.T) {
275302
t.Fatal(err)
276303
}
277304
}
305+
306+
func MemcachedClusterTest(t *testing.T) {
307+
// get global framework variables
308+
f := framework.Global
309+
ctx := f.NewTestCtx(t)
310+
defer ctx.Cleanup(t)
311+
312+
// create rbac
313+
rbacYAML, err := ioutil.ReadFile("deploy/rbac.yaml")
314+
err = ctx.CreateFromYAML(rbacYAML)
315+
if err != nil {
316+
t.Fatalf("failed to create rbac: %v", err)
317+
}
318+
t.Log("Created rbac")
319+
320+
namespace, err := ctx.GetNamespace()
321+
if err != nil {
322+
t.Fatalf("could not get namespace: %v", err)
323+
}
324+
cmdOut, err := exec.Command("operator-sdk", "test", "cluster", *f.ImageName, "-n", namespace, "-i").CombinedOutput()
325+
if err != nil {
326+
t.Fatalf("in-cluster test failed: %v\nCommand Output:\n%s", err, string(cmdOut))
327+
}
328+
329+
}

‎test/test-framework/memcached_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ func memcachedScaleTest(t *testing.T, f *framework.Framework, ctx *framework.Tes
7474
if err != nil {
7575
return err
7676
}
77+
ctx.AddFinalizerFn(func() error {
78+
return f.DynamicClient.Delete(goctx.TODO(), exampleMemcached)
79+
})
7780
// wait for example-memcached to reach 3 replicas
7881
err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-memcached", 3, retryInterval, timeout)
7982
if err != nil {

0 commit comments

Comments
 (0)