Add CRC startup and deployment automation

Signed-off-by: perdasilva <[email protected]>

Author: perdasilva

Makefile

@@ -27,6 +27,10 @@ OPM := $(addprefix bin/, opm)
 OLM_CMDS  := $(shell go list -mod=vendor $(OLM_PKG)/cmd/...)
 PSM_CMD := $(addprefix bin/, psm)
 REGISTRY_CMDS  := $(addprefix bin/, $(shell ls staging/operator-registry/cmd | grep -v opm))
+
+# Default image tag for build/olm-container and build/registry-container
+IMG ?= test:test
+
 # Phony prerequisite for targets that rely on the go build cache to determine staleness.
 .PHONY: FORCE
 FORCE:
@@ -84,10 +88,10 @@ ifeq ($(shell go env GOARCH),amd64)
 endif
 
 build/olm-container:
-	$(CONTAINER_ENGINE) build -f operator-lifecycle-manager.Dockerfile -t test:test .
+	$(CONTAINER_ENGINE) build -f operator-lifecycle-manager.Dockerfile -t ${IMG} .
 
 build/registry-container:
-	$(CONTAINER_ENGINE) build -f operator-registry.Dockerfile -t test:test .
+	$(CONTAINER_ENGINE) build -f operator-registry.Dockerfile -t ${IMG} .
 
 bin/kubebuilder:
 	$(ROOT_DIR)/scripts/install_kubebuilder.sh
@@ -162,6 +166,27 @@ verify:
 	echo "Checking commit integrity"
 	$(MAKE) verify-commits
 
+.PHONY: crc-start
+crc-start:
+	echo "Starting CRC"
+	./scripts/crc-start.sh
+
+.PHONY: crc-build
+crc-build:
+	echo "Building olm image"
+	IMG="olm:test" $(MAKE) build/olm-container
+	echo "Building opm image"
+	rm -rf bin
+	IMG="opm:test" $(MAKE) build/registry-container
+
+.PHONY: crc-deploy
+crc-deploy:
+	echo "Deploying OLM"
+	./scripts/crc-deploy.sh
+
+.PHONY: crc
+crc: crc-start crc-build crc-deploy
+
 .PHONY: help
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

docs/downstream-ci.md

@@ -0,0 +1,133 @@
+# Downstream CI
+
+The CI configuration for each release branch can be found [here](https://github.com/openshift/release/tree/master/ci-operator/config/openshift/operator-framework-olm).
+From `4.11` (`master` as of this writing) we've updated the configuration to able to influence CI on a PR basis. An overview of the `ci-operator` (the system used for ci)
+can be found [here](https://docs.ci.openshift.org/docs/architecture/ci-operator/).
+
+### Structure
+
+ * `.ci-operator.yaml` defines the `build_root_image`. To be ART compliant, the image should come from the [ocp-build-data](https://github.com/openshift/ocp-build-data/) repo
+ * [openshift-operator-framework-olm-master.yaml](https://github.com/openshift/release/blob/master/ci-operator/config/openshift/operator-framework-olm/openshift-operator-framework-olm-master.yaml) defines the images that are used by ci, produced by ci, and the ci jobs the get executed.
+ * `base.Dockerfile` defines the image used by ci to execute the ci jobs
+
+From [openshift-operator-framework-olm-master.yaml](https://github.com/openshift/release/blob/master/ci-operator/config/openshift/operator-framework-olm/openshift-operator-framework-olm-master.yaml), we see under the `images` stanza the `ci-image` definition.
+It goes from `src` (the `build_root_image`) to `ci-image` by building `base.Dockerfile` with `src` as the base image.
+
+```
+- dockerfile_path: base.Dockerfile
+  from: src
+  to: ci-image
+```
+
+The image is excluded from promotion, to never be posted up anywhere:
+
+```
+promotion:
+  excluded_images:
+  - ci-image
+```
+
+and each `test` references `ci-image` as the image to be used to the test, e.g.:
+
+```
+tests:
+- as: verify
+  commands: make verify
+  container:
+    from: ci-image
+```
+
+### Updating go versions
+
+All we need to do is update the `build_root_image` referenced in `.ci-operator.yaml` and we may also need to update the `base_images` in [openshift-operator-framework-olm-master.yaml](https://github.com/openshift/release/blob/master/ci-operator/config/openshift/operator-framework-olm/openshift-operator-framework-olm-master.yaml). 
+
+**NOTE**: I believe there is some automation that updates the base images, though I don't know. I'll leave this as a questions to the reviewer, and if no one knows, I'll go after it.
+
+### Downstream sync
+
+The complete information about the downstreaming process can be found [here](https://docs.google.com/document/d/139yXeOqAJbV1ndC7Q4NbaOtzbSdNpcuJan0iemORd3g/edit).
+
+TL;DR;
+
+We sync three upstream repositories ([api](https://github.com/operator-framework/api), [registry](https://github.com/operator-framework/operator-registry), [olm](https://github.com/operator-framework/operator-lifecycle-manager)) to the downstream [olm mono-repo](https://github.com/openshift/operator-framework-olm). Commits from the upstream repositories are cherry-picked to the appropriate `staging` directory in the downstream repository. Because this is a monorepo in the `Openshift` GitHub organization, two things need to be remembered:
+ - we don't pull in upstream `vendor` folder changes
+ - we don't pull in changes to `OWNERS` files
+ - after each cherry-pick we execute: `make vendor` and `make manifest` to ensure a) the downstream dependencies are updated b) to ensure any manifest changes are picked up downstream
+
+ While manual changes to the `staging` directory should be avoided, there could be instances where there drift between the downstream `staging` directory and the corresponding upstream repository. This can happen due to applying commits out-of-order, e.g. due to feature freeze, etc.
+
+ Therefore, after a sync, it is important to manually verify the diff of `staging` and the upstream. Please note, though, that some downstream changes are downstream only. These are, however, few and far between and there are comments to indicate that a block of code is downstream only.
+
+ The downstream sync process is facilitated by two scripts: `scripts/sync_get_candidates.sh` and `scripts/sync_pop_candidate.sh`, which compare the upstream remote with the appropriate `staging` directory and gets a stack of commits to sync, and cherry-pick those commits in reverse order. What does this look like in practice:
+
+ ```bash
+# Clone downstream
+git clone [email protected]:openshift/operator-framework-olm.git && cd operator-framework-olm
+
+# Add and fetch upstream remotes
+git remote add api [email protected]:operator-framework/api.git && git fetch api
+git remote add operator-registry [email protected]:operator-framework/operator-registry.git && git fetch operator-registry
+git remote add operator-lifecycle-manager [email protected]:operator-framework/operator-lifecycle-manager.git && git fetch operator-lifecycle-manager
+
+# Get upstream commit candidates: ./scripts/sync_get_candidates.sh <api|operator-registry|operator-lifecycle-manager> <branch>
+# The shas will be found in ./<api|operator-registry|operator-lifecycle-manager>.cherrypick
+./scripts/sync_get_candidates.sh api master 
+./scripts/sync_get_candidates.sh operator-registry master 
+./scripts/sync_get_candidates.sh operator-lifecycle-manager master
+
+# Sync upstream commits: ./scripts/sync_pop_candidate.sh <api|operator-registry|operator-lifecycle-manager> [-a]
+# Without -a, you'll proceed one commit at a time. With -a the process will conclude once there are no more commits.
+# When a cherry pick encounters a conflict the script will stop so you can manually fix it.
+sync_pop_candidate.sh operator-lifecycle-manager -a
+
+# When finished
+sync_pop_candidate.sh api -a
+
+# When finished
+sync_pop_candidate.sh operator-registry -a
+
+# Depending on the changes being pulled in, the order of repos you sync _could_ matter and _could_ leave a commit in an unbuildable state
+ ```
+
+ Example: 
+
+ ```bash
+$ sync_pop_candidate.sh operator-lifecycle-manager -a
+
+.github/workflows: Enable workflow_dispatch event triggers (#2464)
+ Author: Tim Flannagan <[email protected]>
+ Date: Mon Dec 20 15:13:33 2021 -0500
+ 9 files changed, 9 insertions(+), 1 deletion(-)
+66 picks remaining (pop_all=true)
+popping: 4daeb114ccd56cee7132883325da68c80ba70bed
+Auto-merging staging/operator-lifecycle-manager/go.mod
+CONFLICT (content): Merge conflict in staging/operator-lifecycle-manager/go.mod
+Auto-merging staging/operator-lifecycle-manager/go.sum
+CONFLICT (content): Merge conflict in staging/operator-lifecycle-manager/go.sum
+CONFLICT (modify/delete): staging/operator-lifecycle-manager/vendor/github.com/operator-framework/api/pkg/validation/doc.go deleted in HEAD and modified in 4daeb114c (chore(api): Vendor the new version of api repo (#2525)).  Version 4daeb114c (chore(api): Vendor the new version of api repo (#2525)) of staging/operator-lifecycle-manager/vendor/github.com/operator-framework/api/pkg/validation/doc.go left in tree.
+CONFLICT (modify/delete): staging/operator-lifecycle-manager/vendor/modules.txt deleted in HEAD and modified in 4daeb114c (chore(api): Vendor the new version of api repo (#2525)).  Version 4daeb114c (chore(api): Vendor the new version of api repo (#2525)) of staging/operator-lifecycle-manager/vendor/modules.txt left in tree.
+error: could not apply 4daeb114c... chore(api): Vendor the new version of api repo (#2525)
+hint: After resolving the conflicts, mark them with
+hint: "git add/rm <pathspec>", then run
+hint: "git cherry-pick --continue".
+hint: You can instead skip this commit with "git cherry-pick --skip".
+hint: To abort and get back to the state before "git cherry-pick",
+hint: run "git cherry-pick --abort".
+
+$ rm -rf staging/operator-lifecycle-manager/vendor
+
+# make sure there are no conflics in 
+# staging/operator-lifecycle-manager/go.mod and go.sum
+$ cd staging/operator-lifecycle-manager
+$ go mod tidy
+$ cd ../../
+
+# now that the conflict is fixed, advance again
+$ sync_pop_candidate.sh operator-lifecycle-manager -a
+ ```
+
+### Troubleshooting
+
+#### Running console test locally
+
+The [console](https://github.com/openshift/console) repository contains all instructions you need to execute the console tests locally. The olm console tests can be found [here](https://github.com/openshift/console/tree/master/frontend/packages/operator-lifecycle-manager)

docs/local-testing-with-crc.md

@@ -0,0 +1,39 @@
+# Local testing with CRC
+
+We can use CRC as an Openshift-like environment within which to test downstream OLM. [CRC](https://developers.redhat.com/products/codeready-containers/overview) 
+is a tool for deploying a local Openshift cluster on your laptop.
+
+TL;DR
+
+1. Install [CRC](https://developers.redhat.com/products/codeready-containers/overview)
+2. `make crc` to provision a CRC cluster, build OLM, and deploy it on the cluster
+3. `make e2e/olm`, i.e., execute e2e tests as you normally would.
+
+#### Gosh darn it, how does it work?
+
+`./scripts/crc-start.sh` is used to provision a crc cluster. `./scripts/crc-deploy.sh` pushes the `olm:test` and `opm:test` to
+`image-registry.openshift-image-registry.svc:5000/openshift/olm:test` and `image-registry.openshift-image-registry.svc:5000/openshift/opm:test`
+images to the crc image registry under the global project `openshift`. Images in the global project are pullable anywhere in the clusters.
+It also generates the olm manifests by applying the `values-crc-e2e.yaml` and other patches (`scripts/*.crc.e2e.patch.yaml`). The manifests are
+deleted (in reverse lexical order, i.e. from deployments, to namespaces, to crds) then recreated (in lexical order). Finally, all olm namespaces
+pods are killed and the script waits until the olm deployments are available.
+
+All deployments were patched to make `ImagePullPolicy: Always`. This way new image tags don't need to be generated and changes can be applied
+by simply killing the pod.
+
+
+#### Make targets
+
+1. `make crc-start`: provision a crc cluster, if necessary
+   1. `FORCE_CLEAN=1 make crc-start`: nuke any current installation including cache and current cluster instance
+2. `make crc-build`: build olm images with the right tags
+3. `make crc-deploy`: generate manifests, upload olm images, deploy olm
+   1. `SKIP_MANIFESTS=1 make crc-deploy`: skip manifest generation and deployment (only update images)
+   2. `SKIP_WAIT_READY=1 make crc-deploy`: skip waiting for olm deployments to be available at the end
+4. `make crc`: the same as `make crc-start crc-build crc-deploy`
+
+#### Manipulating Resources
+
+If new resources are introduced that require being updated for local deployment (e.g. updating the pod spec image), either
+update the e2e-values.yaml at the root, or introduce a yaml patch, e.g. `scripts/psm-operator-deployment.crc.e2e.patch.yaml`
+then update `scripts/generate_crd_manifests.sh` to apply it if the `CRC_E2E` environment variable is set.

scripts/collect-profiles.crc.e2e.patch.yaml

@@ -0,0 +1,6 @@
+- command: update
+  path: spec.jobTemplate.spec.template.spec.containers[0].image
+  value: image-registry.openshift-image-registry.svc:5000/openshift/olm:test
+- command: update
+  path: spec.jobTemplate.spec.template.spec.containers[0].imagePullPolicy
+  value: Always

scripts/crc-deploy.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+set -e
+
+# This script manages the deployment of downstream olm on a local CRC cluster
+# It will deploy the local images: olm:test and opm:test
+# Both built with make build/olm-container and build/registry-container respectively
+
+echo "Deploying OLM to CRC"
+
+# push_images opens a kubectl port-forward session to the crc registry service
+# appropriately tags the locally build olm images
+# and pushes them to the global registry "openshift"
+function push_images {
+  # local images
+  LOCAL_OLM_IMAGE=${1-olm:test}
+  LOCAL_OPM_IMAGE=${2-opm:test}
+
+  # push images to the global crc registry "openshift"
+  # so that they can be pulled from any other namespace in the cluster
+  CRC_GLOBAL_REGISTRY="openshift"
+
+  # CRC destined images
+  CRC_OLM_IMAGE="localhost:5000/${CRC_GLOBAL_REGISTRY}/olm:test"
+  CRC_OPM_IMAGE="localhost:5000/${CRC_GLOBAL_REGISTRY}/opm:test"
+
+  # CRC registry coordinates
+  OPENSHIFT_REGISTRY_NAMESPACE="openshift-image-registry"
+  OLM_NAMESPACE="openshift-operator-lifecycle-manager"
+  IMAGE_REGISTRY_SVC="image-registry"
+  IMAGE_REGISTRY_PORT=5000
+
+  # Create port-forward to CRC registry
+  kubectl port-forward -n ${OPENSHIFT_REGISTRY_NAMESPACE} svc/${IMAGE_REGISTRY_SVC} ${IMAGE_REGISTRY_PORT}:${IMAGE_REGISTRY_PORT} > /dev/null 2>&1 &
+  PORT_FWD_PID=$!
+
+  # Remember to close the port-forward
+  trap 'kill "${PORT_FWD_PID}"' EXIT
+
+  # give port-forward a second
+  # I found that without this docker login would not work
+  # as if it couldn't reach the docker server
+  sleep 1
+
+  # Login to the CRC registry
+  oc whoami -t | docker login localhost:5000 --username user --password-stdin
+
+  # Tag and push olm image
+  echo "Pushing olm image"
+  docker tag "${LOCAL_OLM_IMAGE}" "${CRC_OLM_IMAGE}"
+  docker push "${CRC_OLM_IMAGE}"
+
+  # Tag and push registry image
+  echo "Pushing registry image"
+  docker tag "${LOCAL_OPM_IMAGE}" "${CRC_OPM_IMAGE}"
+  docker push "${CRC_OPM_IMAGE}"
+}
+
+# Set kubeconfig to CRC if necessary
+export KUBECONFIG=${KUBECONFIG:-${HOME}/.crc/machines/crc/kubeconfig}
+KUBE_ADMIN_LOGIN=$(crc console --credentials -o json | jq -r .clusterConfig.adminCredentials.username)
+KUBE_ADMIN_PASSWORD=$(crc console --credentials -o json | jq -r .clusterConfig.adminCredentials.password)
+
+# login to crc
+echo "Logging in as kubeadmin"
+oc login -u "${KUBE_ADMIN_LOGIN}" -p "${KUBE_ADMIN_PASSWORD}" > /dev/null
+
+# Scale down CVO to stop changes from returning to stock configuration
+echo "Scaling down CVO"
+oc scale --replicas 0 -n openshift-cluster-version deployments/cluster-version-operator
+
+# push olm images to crc global repository
+push_images olm:test opm:test
+
+SKIP_MANIFESTS=${SKIP_MANIFESTS:-0}
+if [ "${SKIP_MANIFESTS}" = 0 ]; then
+  # Build e2e manifests
+  echo "Generating manifests"
+  CRC_E2E=1 ./scripts/generate_crds_manifests.sh
+
+  # Use the numbered ordering in the manifest file names to delete and deploy the manifests in order
+  # First in reverse with sort -r to delete then in order to recreate
+  echo "Deploying OLM"
+  find_flags=(-regex ".*\.yaml" -not -regex ".*\.removed\.yaml" -not -regex ".*\.ibm-cloud-managed\.yaml")
+
+  echo "Replacing manifests"
+  find manifests "${find_flags[@]}" | while read -r manifest; do
+    echo "Deleting ${manifest}"
+    set +e
+    kubectl replace -f "${manifest}"
+    set -e
+  done
+fi
+
+# Restart olm pods
+# This is required because the images were pushed after the olm resources were applied
+# So we need to ensure the new images are picked up by the deployments
+echo "Restarting OLM pods"
+for POD in $(oc get pods --no-headers=true -n "${OLM_NAMESPACE}" | awk '{ print $1 }'); do
+  oc delete pod --ignore-not-found=true -n ${OLM_NAMESPACE} "${POD}"
+done
+
+# Wait for deployments to be available
+SKIP_WAIT_READY=${SKIP_WAIT_READY:-0}
+if [ "${SKIP_WAIT_READY}" = 0 ]; then
+  echo "Waiting on deployments to be ready"
+  for DEPLOYMENT in $(oc get deployments --no-headers=true -n "${OLM_NAMESPACE}" | awk '{ print $1 }'); do
+    echo "Waiting for ${DEPLOYMENT}"
+    kubectl wait --for=condition=available --timeout=120s "deployment/${DEPLOYMENT}" -n "${OLM_NAMESPACE}"
+  done
+fi
+
+echo "Done"
+
+exit 0

scripts/crc-start.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+
+# This script manages the creation of the CRC cluster to be used for testing
+# Usage: crc-start.sh
+# FORCE_CLEAN=1 crc-start.sh will delete any current crc cluster, clear the cache and start a fresh installation
+# If CRC is already running, nothing happens
+
+# Check CRC is installed
+if ! [ -x "$(command -v crc)" ]; then
+  echo "Error: CRC is not installed. Go to: https://developers.redhat.com/products/codeready-containers/overview"
+  exit 1
+fi
+
+# Blast CRC if necessary
+if [ "${FORCE_CLEAN}" = 1 ]; then
+  crc delete --clear-cache --force
+fi
+
+# Start CRC if necessary
+if [ "$(crc status -o json | jq -r .success)" = "false"  ] || [ "$(crc status -o json | jq -r .crcStatus)" = "Stopped" ]; then
+    echo "Setting up CRC"
+    crc setup
+    crc start
+fi
+
+# Check CRC started successfully
+if ! [ "$(crc status -o json | jq -r .crcStatus)" = "Running" ]; then
+  echo "Error: CRC is unreachable. Please try recreating the cluster."
+  exit 1
+fi
+
+echo "SUCCESS! kubeconfig=${HOME}/.crc/machines/crc/kubeconfig"