ci,test,hack/tests: add openshift-ci e2e support for subcommands (#1654)

Author: AlexNPavel

Makefile

@@ -110,12 +110,18 @@ test/unit:
 
 test/subcommand: test/subcommand/test-local test/subcommand/scorecard test/subcommand/alpha-olm
 
+test/subcommand2:
+	./ci/tests/subcommand.sh
+
 test/subcommand/test-local:
 	./hack/tests/test-subcommand.sh
 
 test/subcommand/scorecard:
 	./hack/tests/scorecard-subcommand.sh
 
+test/subcommand/scorecard2:
+	./ci/tests/scorecard-subcommand.sh
+
 test/subcommand/alpha-olm:
 	./hack/tests/alpha-olm-subcommands.sh
 

ci/dockerfiles/scorecard-proxy.Dockerfile

@@ -0,0 +1,19 @@
+FROM osdk-builder as builder
+
+RUN ci/tests/scorecard-proxy-scaffold.sh
+
+FROM registry.access.redhat.com/ubi7/ubi-minimal:latest
+
+ENV PROXY=/usr/local/bin/scorecard-proxy \
+    USER_UID=1001 \
+    USER_NAME=proxy
+
+# install operator binary
+COPY --from=builder /scorecard/scorecard-proxy ${PROXY}
+
+COPY --from=builder /scorecard/bin /usr/local/bin
+RUN  /usr/local/bin/user_setup
+
+ENTRYPOINT ["/usr/local/bin/entrypoint"]
+
+USER ${USER_UID}

ci/tests/scorecard-proxy-scaffold.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -eux
+
+ROOTDIR="$(pwd)"
+SCORECARD_DIR="/scorecard"
+
+mkdir -p $SCORECARD_DIR
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
+  go build \
+  -gcflags "all=-trimpath=${ROOTDIR}" \
+  -asmflags "all=-trimpath=${ROOTDIR}" \
+  -o $SCORECARD_DIR/scorecard-proxy \
+  images/scorecard-proxy/cmd/proxy/main.go
+mv images/scorecard-proxy/bin $SCORECARD_DIR/bin

ci/tests/scorecard-subcommand.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+component="scorecard-proxy"
+eval IMAGE=$IMAGE_FORMAT
+CSV_PATH="deploy/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml"
+CONFIG_PATH=".test-osdk-scorecard.yaml"
+
+set -ex
+
+# the test framework directory has all the manifests needed to run the cluster
+pushd test/test-framework
+commandoutput="$(operator-sdk scorecard \
+  --cr-manifest deploy/crds/cache_v1alpha1_memcached_cr.yaml \
+  --cr-manifest deploy/crds/cache_v1alpha1_memcachedrs_cr.yaml \
+  --init-timeout 60 \
+  --csv-path "$CSV_PATH" \
+  --verbose \
+  --proxy-image "$IMAGE" \
+  2>&1)"
+echo $commandoutput | grep "Total Score: 82%"
+
+# test config file
+commandoutput2="$(operator-sdk scorecard \
+  --proxy-image "$IMAGE" \
+  --config "$CONFIG_PATH")"
+# check basic suite
+echo $commandoutput2 | grep '^.*"error": 0,[[:space:]]"pass": 3,[[:space:]]"partialPass": 0,[[:space:]]"fail": 0,[[:space:]]"totalTests": 3,[[:space:]]"totalScorePercent": 100,.*$'
+# check olm suite
+echo $commandoutput2 | grep '^.*"error": 0,[[:space:]]"pass": 2,[[:space:]]"partialPass": 3,[[:space:]]"fail": 0,[[:space:]]"totalTests": 5,[[:space:]]"totalScorePercent": 74,.*$'
+# check custom json result
+echo $commandoutput2 | grep '^.*"error": 0,[[:space:]]"pass": 1,[[:space:]]"partialPass": 1,[[:space:]]"fail": 0,[[:space:]]"totalTests": 2,[[:space:]]"totalScorePercent": 71,.*$'
+popd

ci/tests/subcommand.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+source hack/lib/test_lib.sh
+
+set -eux
+
+ROOTDIR="$(pwd)"
+mkdir -p $ROOTDIR/bin
+export PATH=$ROOTDIR/bin:$PATH
+
+if ! [ -x "$(command -v kubectl)" ]; then
+    curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.14.2/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl bin/
+fi
+
+if ! [ -x "$(command -v oc)" ]; then
+    curl -Lo oc.tar.gz https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz
+    tar xvzOf oc.tar.gz openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit/oc > oc && chmod +x oc && mv oc bin/ && rm oc.tar.gz
+fi
+
+oc version
+
+make install
+
+./hack/tests/test-subcommand.sh
+./ci/tests/scorecard-subcommand.sh
+# This can't run in openshift-ci because the test cluster already has OLM installed.
+#./hack/tests/alpha-olm-subcommands.sh

hack/tests/scorecard-subcommand.sh

@@ -25,6 +25,7 @@ echo $commandoutput | grep "Total Score: 82%"
 # test config file
 commandoutput2="$(operator-sdk scorecard \
   --proxy-image "$DEST_IMAGE" \
+  --proxy-pull-policy "Never" \
   --config "$CONFIG_PATH")"
 # check basic suite
 echo $commandoutput2 | grep '^.*"error": 0,[[:space:]]"pass": 3,[[:space:]]"partialPass": 0,[[:space:]]"fail": 0,[[:space:]]"totalTests": 3,[[:space:]]"totalScorePercent": 100,.*$'

hack/tests/test-subcommand.sh

@@ -3,12 +3,16 @@ source hack/lib/test_lib.sh
 
 set -ex
 
+if [ -z "$KUBECONFIG" ]; then
+  KUBECONFIG=$HOME/.kube/config
+fi
+
 pushd test/test-framework
 # test framework with defaults
 operator-sdk test local ./test/e2e
 
 # test operator-sdk test flags
-operator-sdk test local ./test/e2e --global-manifest deploy/crds/cache_v1alpha1_memcached_crd.yaml --namespaced-manifest deploy/namespace-init.yaml --go-test-flags "-parallel 1" --kubeconfig $HOME/.kube/config --image=quay.io/coreos/operator-sdk-dev:test-framework-operator-runtime
+operator-sdk test local ./test/e2e --global-manifest deploy/crds/cache_v1alpha1_memcached_crd.yaml --namespaced-manifest deploy/namespace-init.yaml --go-test-flags "-parallel 1" --kubeconfig $KUBECONFIG --image=quay.io/coreos/operator-sdk-dev:test-framework-operator-runtime
 
 # we use the test-memcached namespace for all future tests, so we only need to set this trap once
 kubectl create namespace test-memcached
@@ -23,7 +27,7 @@ kubectl delete namespace test-memcached
 
 # test operator in up local mode with kubeconfig
 kubectl create namespace test-memcached
-operator-sdk test local ./test/e2e --up-local --namespace=test-memcached --kubeconfig $HOME/.kube/config
+operator-sdk test local ./test/e2e --up-local --namespace=test-memcached --kubeconfig $KUBECONFIG
 kubectl delete namespace test-memcached
 
 # test operator in no-setup mode

test/test-framework/.test-osdk-scorecard.yaml

@@ -3,7 +3,6 @@ cr-manifest:
   - "deploy/crds/cache_v1alpha1_memcachedrs_cr.yaml"
 init-timeout: 60
 csv-path: "deploy/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml"
-proxy-image: "scorecard-proxy"
-proxy-pull-policy: "Never"
 verbose: true
 output: json
+proxy-image: "scorecard-proxy"

test/test-framework/deploy/namespace-init.yaml

@@ -65,6 +65,7 @@ roleRef:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  creationTimestamp: null
   name: memcached-operator
 spec:
   replicas: 1
@@ -74,6 +75,7 @@ spec:
   strategy: {}
   template:
     metadata:
+      creationTimestamp: null
       labels:
         name: memcached-operator
     spec: