Fix the version checker job update errors out with field is immutable

Currently, In version checker job we reconcile the job and create the job if not present and update the job if there is some change in existing job. However updating the template of the job is not allowed and we get the "Field is immutable" error. I have fixed it by recreating the job instead of updating it.

Author: prafull01

.bazelrc

@@ -1,2 +1,2 @@
 build --workspace_status_command hack/build/print-workspace-status.sh
-test --test_output=errors --test_timeout=-1,-1,-1,2400
+test --test_output=errors --test_timeout=-1,-1,-1,3600

hack/bin/BUILD.bazel

@@ -42,6 +42,7 @@ genrule(
     name = "fetch_preflight",
     srcs = select({
         ":k8": ["@preflight_linux//file"],
+        ":m1": ["@preflight_linux//file"],
     }),
     outs = ["preflight"],
     cmd = "cp $(SRCS) $@",

pkg/actor/validate_version.go

@@ -86,7 +86,7 @@ func (v *versionChecker) Act(ctx context.Context, cluster *resource.Cluster, log
 
 	log.V(DEBUGLEVEL).Info("starting to check the crdb version of the container provided")
 
-	r := resource.NewManagedKubeResource(ctx, v.client, cluster, kube.AnnotatingPersister)
+	r := resource.NewManagedKubeResource(ctx, v.client, cluster, kube.RecreatingPersister)
 	owner := cluster.Unwrap()
 
 	// If the image.name is set use that value and do not check that the

pkg/kube/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach-operator/pkg/kube",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/ptr:go_default_library",
         "@com_github_banzaicloud_k8s_objectmatcher//patch:go_default_library",
         "@com_github_cenkalti_backoff//:go_default_library",
         "@com_github_cockroachdb_errors//:go_default_library",

pkg/kube/helpers.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/banzaicloud/k8s-objectmatcher/patch"
 	"github.com/cenkalti/backoff"
+	"github.com/cockroachdb/cockroach-operator/pkg/ptr"
 	"github.com/cockroachdb/errors"
 	"github.com/go-logr/logr"
 	"go.uber.org/zap/zapcore"
@@ -140,6 +141,12 @@ var AnnotatingPersister PersistFn = func(ctx context.Context, cl client.Client,
 	})
 }
 
+var RecreatingPersister PersistFn = func(ctx context.Context, cl client.Client, obj client.Object, f MutateFn) (upserted bool, err error) {
+	return ReCreateAnnotated(ctx, cl, obj, func() error {
+		return f()
+	})
+}
+
 // MutateFn is a function which mutates the existing object into it's desired state.
 type MutateFn func() error
 
@@ -190,6 +197,49 @@ func CreateOrUpdateAnnotated(ctx context.Context, c client.Client, obj client.Ob
 	return true, nil
 }
 
+func ReCreateAnnotated(ctx context.Context, c client.Client, obj client.Object, f MutateFn) (upserted bool, err error) {
+	key := client.ObjectKeyFromObject(obj)
+
+	if err := c.Get(ctx, key, obj); err == nil {
+		existing := obj.DeepCopyObject()
+		if err := mutate(f, key, obj); err != nil {
+			return false, err
+		}
+
+		changed, err := ObjectChanged(existing, obj)
+		if err != nil {
+			return false, err
+		}
+		if !changed {
+			return false, nil
+		}
+
+		dp := metav1.DeletePropagationForeground
+		if err := c.Delete(ctx, existing.(client.Object), &client.DeleteOptions{GracePeriodSeconds: ptr.Int64(0),
+			PropagationPolicy: &dp}); err != nil {
+			return false, err
+		}
+
+		return false, nil
+	} else if err != nil && !apierrors.IsNotFound(err) {
+		return false, err
+	}
+
+	if err := mutate(f, key, obj); err != nil {
+		return false, err
+	}
+
+	if err := annotator.SetLastAppliedAnnotation(obj); err != nil {
+		return false, err
+	}
+
+	if err := c.Create(ctx, obj); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
 func ObjectChanged(current, updated runtime.Object) (bool, error) {
 	opts := []patch.CalculateOption{
 		patch.IgnoreStatusFields(),