Fix the version checker job update errors out with field is immutable

Currently, In version checker job we reconcile the job and create the job if not present and update the job if there is some change in existing job. However updating the template of the job is not allowed and we get the "Field is immutable" error. I have fixed it by recreating the job instead of updating it.

Author: prafull01

.bazelrc

@@ -1,2 +1,2 @@
 build --workspace_status_command hack/build/print-workspace-status.sh
-test --test_output=errors --test_timeout=-1,-1,-1,2400
+test --test_output=errors --test_timeout=-1,-1,-1,3600

config/manager/patches/image.yaml

@@ -218,3 +218,5 @@ spec:
               value: cockroachdb/cockroach:v23.1.1
             - name: RELATED_IMAGE_COCKROACH_v23_1_2
               value: cockroachdb/cockroach:v23.1.2
+            - name: RELATED_IMAGE_COCKROACH_v23_1_3
+              value: cockroachdb/cockroach:v23.1.3

config/manifests/bases/cockroach-operator.clusterserviceversion.yaml

@@ -375,3 +375,5 @@ spec:
     image: registry.connect.redhat.com/cockroachdb/cockroach@sha256:bc4b75ed71845d5b18e1d9163d480fd9d0d3cb5ebbcfed00a2e4e1b174c0a5de
   - name: RELATED_IMAGE_COCKROACH_v23_1_2
     image: registry.connect.redhat.com/cockroachdb/cockroach@sha256:0b0e0588c1aadf0c3826cd584bd2b3a7d6781882dcb5c5e037b47ff5cb84509b
+  - name: RELATED_IMAGE_COCKROACH_v23_1_3
+    image: registry.connect.redhat.com/cockroachdb/cockroach@sha256:40f00476b63cb9e370fe85f01407173693213a9242fc117d28b06378ca0d98e0

config/manifests/patches/deployment_patch.yaml

@@ -227,4 +227,6 @@ spec:
               value: registry.connect.redhat.com/cockroachdb/cockroach@sha256:bc4b75ed71845d5b18e1d9163d480fd9d0d3cb5ebbcfed00a2e4e1b174c0a5de
             - name: RELATED_IMAGE_COCKROACH_v23_1_2
               value: registry.connect.redhat.com/cockroachdb/cockroach@sha256:0b0e0588c1aadf0c3826cd584bd2b3a7d6781882dcb5c5e037b47ff5cb84509b
+            - name: RELATED_IMAGE_COCKROACH_v23_1_3
+              value: registry.connect.redhat.com/cockroachdb/cockroach@sha256:40f00476b63cb9e370fe85f01407173693213a9242fc117d28b06378ca0d98e0
           image: RH_COCKROACH_OP_IMAGE_PLACEHOLDER

config/samples/crdb-tls-example.yaml

@@ -19,7 +19,7 @@ kind: CrdbCluster
 metadata:
   name: crdb-tls-example
 spec:
-  cockroachDBVersion: v23.1.2
+  cockroachDBVersion: v23.1.3
   dataStore:
     pvc:
       spec:

crdb-versions.yaml

@@ -310,3 +310,6 @@ CrdbVersions:
 - image: cockroachdb/cockroach:v23.1.2
   redhatImage: registry.connect.redhat.com/cockroachdb/cockroach@sha256:0b0e0588c1aadf0c3826cd584bd2b3a7d6781882dcb5c5e037b47ff5cb84509b
   tag: v23.1.2
+- image: cockroachdb/cockroach:v23.1.3
+  redhatImage: registry.connect.redhat.com/cockroachdb/cockroach@sha256:40f00476b63cb9e370fe85f01407173693213a9242fc117d28b06378ca0d98e0
+  tag: v23.1.3

examples/client-secure-operator.yaml

@@ -23,7 +23,7 @@ spec:
   serviceAccountName: cockroachdb-sa
   containers:
   - name: cockroachdb-client-secure
-    image: cockroachdb/cockroach:v23.1.2
+    image: cockroachdb/cockroach:v23.1.3
     imagePullPolicy: IfNotPresent
     volumeMounts:
     - name: client-certs

examples/example.yaml

@@ -40,9 +40,9 @@ spec:
       memory: 8Gi
   tlsEnabled: true
 # You can set either a version of the db or a specific image name
-# cockroachDBVersion: v23.1.2
+# cockroachDBVersion: v23.1.3
   image:
-    name: cockroachdb/cockroach:v23.1.2
+    name: cockroachdb/cockroach:v23.1.3
   # nodes refers to the number of crdb pods that are created
   # via the statefulset
   nodes: 3

examples/smoketest.yaml

@@ -39,5 +39,5 @@ spec:
       memory: 300Mi
   tlsEnabled: true
   image:
-    name: cockroachdb/cockroach:v23.1.2
+    name: cockroachdb/cockroach:v23.1.3
   nodes: 3

hack/bin/BUILD.bazel

@@ -42,6 +42,7 @@ genrule(
     name = "fetch_preflight",
     srcs = select({
         ":k8": ["@preflight_linux//file"],
+        ":m1": ["@preflight_linux//file"],
     }),
     outs = ["preflight"],
     cmd = "cp $(SRCS) $@",

install/operator.yaml

@@ -576,6 +576,8 @@ spec:
           value: cockroachdb/cockroach:v23.1.1
         - name: RELATED_IMAGE_COCKROACH_v23_1_2
           value: cockroachdb/cockroach:v23.1.2
+        - name: RELATED_IMAGE_COCKROACH_v23_1_3
+          value: cockroachdb/cockroach:v23.1.3
         - name: OPERATOR_NAME
           value: cockroachdb
         - name: POD_NAME

pkg/actor/validate_version.go

@@ -86,7 +86,7 @@ func (v *versionChecker) Act(ctx context.Context, cluster *resource.Cluster, log
 
 	log.V(DEBUGLEVEL).Info("starting to check the crdb version of the container provided")
 
-	r := resource.NewManagedKubeResource(ctx, v.client, cluster, kube.AnnotatingPersister)
+	r := resource.NewManagedKubeResource(ctx, v.client, cluster, kube.RecreatingPersister)
 	owner := cluster.Unwrap()
 
 	// If the image.name is set use that value and do not check that the

pkg/kube/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach-operator/pkg/kube",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/ptr:go_default_library",
         "@com_github_banzaicloud_k8s_objectmatcher//patch:go_default_library",
         "@com_github_cenkalti_backoff//:go_default_library",
         "@com_github_cockroachdb_errors//:go_default_library",

pkg/kube/helpers.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/banzaicloud/k8s-objectmatcher/patch"
 	"github.com/cenkalti/backoff"
+	"github.com/cockroachdb/cockroach-operator/pkg/ptr"
 	"github.com/cockroachdb/errors"
 	"github.com/go-logr/logr"
 	"go.uber.org/zap/zapcore"
@@ -140,6 +141,12 @@ var AnnotatingPersister PersistFn = func(ctx context.Context, cl client.Client,
 	})
 }
 
+var RecreatingPersister PersistFn = func(ctx context.Context, cl client.Client, obj client.Object, f MutateFn) (upserted bool, err error) {
+	return ReCreateAnnotated(ctx, cl, obj, func() error {
+		return f()
+	})
+}
+
 // MutateFn is a function which mutates the existing object into it's desired state.
 type MutateFn func() error
 
@@ -190,6 +197,49 @@ func CreateOrUpdateAnnotated(ctx context.Context, c client.Client, obj client.Ob
 	return true, nil
 }
 
+func ReCreateAnnotated(ctx context.Context, c client.Client, obj client.Object, f MutateFn) (upserted bool, err error) {
+	key := client.ObjectKeyFromObject(obj)
+
+	if err := c.Get(ctx, key, obj); err == nil {
+		existing := obj.DeepCopyObject()
+		if err := mutate(f, key, obj); err != nil {
+			return false, err
+		}
+
+		changed, err := ObjectChanged(existing, obj)
+		if err != nil {
+			return false, err
+		}
+		if !changed {
+			return false, nil
+		}
+
+		dp := metav1.DeletePropagationForeground
+		if err := c.Delete(ctx, existing.(client.Object), &client.DeleteOptions{GracePeriodSeconds: ptr.Int64(0),
+			PropagationPolicy: &dp}); err != nil {
+			return false, err
+		}
+
+		return false, nil
+	} else if err != nil && !apierrors.IsNotFound(err) {
+		return false, err
+	}
+
+	if err := mutate(f, key, obj); err != nil {
+		return false, err
+	}
+
+	if err := annotator.SetLastAppliedAnnotation(obj); err != nil {
+		return false, err
+	}
+
+	if err := c.Create(ctx, obj); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
 func ObjectChanged(current, updated runtime.Object) (bool, error) {
 	opts := []patch.CalculateOption{
 		patch.IgnoreStatusFields(),