Rework config API

.github/workflows/verify-build.yaml

@@ -20,13 +20,21 @@ on:
 jobs:
   verify:
     runs-on: ubuntu-latest
+    env:
+      GOPATH: /home/runner/work/kubernetes-controller-sharding/kubernetes-controller-sharding/go
+    defaults:
+      run:
+        working-directory: go/src/github.com/timebertt/kubernetes-controller-sharding
+
     steps:
     - uses: actions/checkout@v3
+      with:
+        path: go/src/github.com/timebertt/kubernetes-controller-sharding
     - name: Setup Go
       uses: actions/setup-go@v4
       with:
-        go-version-file: webhosting-operator/go.mod
-        cache-dependency-path: webhosting-operator/go.sum
+        go-version-file: go/src/github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/go.mod
+        cache-dependency-path: go/src/github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/go.sum
     - name: Verify
       run: make -C webhosting-operator verify
 

webhosting-operator/.run/webhosting-operator process (kind).run.xml

@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="webhosting-operator process (kind)" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="kubernetes-controller-sharding" />
+    <working_directory value="$PROJECT_DIR$/webhosting-operator" />
+    <envs>
+      <env name="KUBECONFIG" value="$PROJECT_DIR$/webhosting-operator/dev/kind_kubeconfig.yaml" />
+      <env name="LEADER_ELECT" value="false" />
+      <env name="SHARD_ID" value="webhosting-operator-0" />
+      <env name="SHARD_MODE" value="sharder" />
+    </envs>
+    <kind value="PACKAGE" />
+    <package value="github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/cmd/webhosting-operator" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$/webhosting-operator/main.go" />
+    <method v="2" />
+  </configuration>
+</component>

webhosting-operator/.run/webhosting-operator.run.xml

@@ -1,20 +1,4 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="webhosting-operator process (kind)" type="GoApplicationRunConfiguration" factoryName="Go Application">
-    <module name="kubernetes-controller-sharding" />
-    <working_directory value="$PROJECT_DIR$/webhosting-operator" />
-    <parameters value="--config=config/manager/default/controller_manager_config.yaml" />
-    <envs>
-      <env name="KUBECONFIG" value="$PROJECT_DIR$/webhosting-operator/dev/kind_kubeconfig.yaml" />
-      <env name="LEADER_ELECT" value="false" />
-      <env name="SHARD_ID" value="webhosting-operator-0" />
-      <env name="SHARD_MODE" value="sharder" />
-    </envs>
-    <kind value="PACKAGE" />
-    <package value="github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/cmd/webhosting-operator" />
-    <directory value="$PROJECT_DIR$" />
-    <filePath value="$PROJECT_DIR$/webhosting-operator/main.go" />
-    <method v="2" />
-  </configuration>
   <configuration default="false" name="webhosting-operator skaffold" type="google-container-tools-skaffold-run-config" factoryName="google-container-tools-skaffold-run-config-dev" show_console_on_std_err="false" show_console_on_std_out="false" kubeconfigFile="{&quot;displayName&quot;:&quot;$PROJECT_DIR$/webhosting-operator/dev/kind_kubeconfig.yaml&quot;,&quot;path&quot;:&quot;$PROJECT_DIR$/webhosting-operator/dev/kind_kubeconfig.yaml&quot;}">
     <option name="allowRunningInParallel" value="false" />
     <option name="buildEnvironment" />
@@ -46,4 +30,4 @@
     <option name="verbosity" value="WARN" />
     <method v="2" />
   </configuration>
-</component>
+</component>

webhosting-operator/Dockerfile

@@ -26,9 +26,9 @@ COPY pkg/ pkg/
 # Build
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-    go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o manager ./cmd/webhosting-operator
+    go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o webhosting-operator ./cmd/webhosting-operator
 
-# Use distroless as minimal base image to package the manager binary
+# Use distroless as minimal base image to package the webhosting-operator binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot AS base
 
@@ -41,5 +41,5 @@ USER 65532:65532
 
 FROM base AS webhosting-operator
 
-COPY --from=builder /workspace/manager .
-ENTRYPOINT ["/manager"]
+COPY --from=builder /workspace/webhosting-operator .
+ENTRYPOINT ["/webhosting-operator"]

webhosting-operator/Makefile

@@ -45,11 +45,12 @@ clean-tools-bin: ## Empty the tools binary directory
 
 .PHONY: manifests
 manifests: $(CONTROLLER_GEN) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=operator crd paths="./..." output:rbac:artifacts:config=config/manager/rbac output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) rbac:roleName=operator crd paths="./..." output:rbac:artifacts:config=config/manager/rbac output:crd:artifacts:config=config/manager/crds
 
 .PHONY: generate
 generate: $(CONTROLLER_GEN) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+	hack/update-codegen.sh
 
 .PHONY: fmt
 fmt: ## Run go fmt against code.
@@ -98,7 +99,7 @@ build: generate fmt vet ## Build manager binary.
 
 .PHONY: run
 run: manifests generate fmt vet ## Run the webhosting-operator from your host.
-	go run ./cmd/webhosting-operator --config=config/manager/default/controller_manager_config.yaml
+	go run ./cmd/webhosting-operator
 
 .PHONY: docker-build
 docker-build: test ## Build docker image with the manager.

webhosting-operator/PROJECT

@@ -28,7 +28,7 @@ resources:
     namespaced: true
   domain: webhosting.timebertt.dev
   group: config
-  kind: ControllerManagerConfig
+  kind: WebhostingOperatorConfig
   path: github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/apis/config/v1alpha1
   version: v1alpha1
 version: "3"

webhosting-operator/cmd/webhosting-operator/main.go

@@ -26,13 +26,16 @@ import (
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/controller/sharding"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -92,18 +95,18 @@ func main() {
 		os.Exit(1)
 	}
 
-	if debugging := opts.controllerManagerConfig.Debugging; debugging != nil && pointer.BoolDeref(debugging.EnableProfiling, false) {
+	if *opts.config.Debugging.EnableProfiling {
 		if err := (routes.Profiling{}).AddToManager(mgr); err != nil {
 			setupLog.Error(err, "failed adding profiling handlers to manager")
 			os.Exit(1)
 		}
-		if pointer.BoolDeref(opts.controllerManagerConfig.Debugging.EnableContentionProfiling, false) {
+		if *opts.config.Debugging.EnableContentionProfiling {
 			goruntime.SetBlockProfileRate(1)
 		}
 	}
 
 	if err = (&webhosting.WebsiteReconciler{
-		Config: opts.controllerManagerConfig,
+		Config: opts.config,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Website")
 		os.Exit(1)
@@ -129,95 +132,115 @@ func main() {
 type options struct {
 	configFile string
 
-	restConfig              *rest.Config
-	managerOptions          ctrl.Options
-	controllerManagerConfig *configv1alpha1.ControllerManagerConfig
+	restConfig     *rest.Config
+	config         *configv1alpha1.WebhostingOperatorConfig
+	managerOptions ctrl.Options
 }
 
 func (o *options) AddFlags(fs *flag.FlagSet) {
 	fs.StringVar(&o.configFile, "config", "",
-		"The controller will load its initial configuration from this file. "+
-			"Omit this flag to use the default configuration values. "+
-			"Command-line flags override configuration from this file.")
+		"Path to the file containing the operator's configuration (WebhostingOperatorConfig). "+
+			"If not specified, the operator will use the default configuration values.")
 }
 
 func (o *options) Complete() error {
-	o.controllerManagerConfig = &configv1alpha1.ControllerManagerConfig{}
+	o.config = &configv1alpha1.WebhostingOperatorConfig{}
 
-	// load manager options
-	var err error
-	opts := ctrl.Options{Scheme: scheme}
+	// load config file if specified
 	if o.configFile != "" {
-		opts, err = opts.AndFrom(ctrl.ConfigFile().AtPath(o.configFile).OfKind(o.controllerManagerConfig))
+		configBytes, err := os.ReadFile(o.configFile)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed reading config file: %w", err)
+		}
+
+		if err := runtime.DecodeInto(serializer.NewCodecFactory(scheme).UniversalDecoder(), configBytes, o.config); err != nil {
+			return fmt.Errorf("failed decoding config file: %w", err)
 		}
+	} else {
+		scheme.Default(o.config)
 	}
 
-	opts, err = applyOptionsOverrides(opts)
+	// load rest config
+	var err error
+	o.restConfig, err = ctrl.GetConfig()
 	if err != nil {
+		return fmt.Errorf("failed loading kubeconfig: %w", err)
+	}
+
+	// bring everything together
+	o.managerOptions = ctrl.Options{
+		Scheme: scheme,
+		// allows us to quickly handover leadership on restarts
+		LeaderElectionReleaseOnCancel: true,
+		Cache: cache.Options{
+			DefaultTransform: dropUnwantedMetadata,
+		},
+		Controller: config.Controller{
+			RecoverPanic: pointer.Bool(true),
+		},
+	}
+
+	o.applyConfigToRESTConfig()
+	o.applyConfigToOptions()
+	if err := o.applyOptionsOverrides(); err != nil {
 		return err
 	}
 
-	// apply some sensible defaults
-	o.managerOptions = setOptionsDefaults(opts)
+	return nil
+}
 
-	// create rest config
-	o.restConfig = ctrl.GetConfigOrDie()
-	// increase default rate limiter settings to make sharder and controller more responsive
-	o.restConfig.QPS = 100
-	o.restConfig.Burst = 150
-	if clientConnection := o.controllerManagerConfig.ClientConnection; clientConnection != nil {
+func (o *options) applyConfigToRESTConfig() {
+	if clientConnection := o.config.ClientConnection; clientConnection != nil {
 		if clientConnection.QPS > 0 {
 			o.restConfig.QPS = clientConnection.QPS
 		}
 		if clientConnection.Burst > 0 {
 			o.restConfig.Burst = int(clientConnection.Burst)
 		}
 	}
+}
 
-	return nil
+func (o *options) applyConfigToOptions() {
+	if leaderElection := o.config.LeaderElection; leaderElection != nil {
+		o.managerOptions.LeaderElection = *leaderElection.LeaderElect
+		o.managerOptions.LeaderElectionResourceLock = leaderElection.ResourceLock
+		o.managerOptions.LeaderElectionID = leaderElection.ResourceName
+		o.managerOptions.LeaderElectionNamespace = leaderElection.ResourceNamespace
+		o.managerOptions.LeaseDuration = pointer.Duration(leaderElection.LeaseDuration.Duration)
+		o.managerOptions.RenewDeadline = pointer.Duration(leaderElection.RenewDeadline.Duration)
+		o.managerOptions.RetryPeriod = pointer.Duration(leaderElection.RetryPeriod.Duration)
+	}
+
+	o.managerOptions.HealthProbeBindAddress = o.config.Health.BindAddress
+	o.managerOptions.MetricsBindAddress = o.config.Metrics.BindAddress
+	o.managerOptions.GracefulShutdownTimeout = pointer.Duration(o.config.GracefulShutdownTimeout.Duration)
 }
 
-func applyOptionsOverrides(opts ctrl.Options) (ctrl.Options, error) {
+func (o *options) applyOptionsOverrides() error {
 	var err error
 
 	// allow overriding leader election via env var for debugging purposes
 	if leaderElectEnv, ok := os.LookupEnv("LEADER_ELECT"); ok {
-		opts.LeaderElection, err = strconv.ParseBool(leaderElectEnv)
+		o.managerOptions.LeaderElection, err = strconv.ParseBool(leaderElectEnv)
 		if err != nil {
-			return ctrl.Options{}, fmt.Errorf("error parsing LEADER_ELECT env var: %w", err)
+			return fmt.Errorf("error parsing LEADER_ELECT env var: %w", err)
 		}
 	}
 
-	opts.Sharded = true
+	o.managerOptions.Sharded = true
 	// allow disabling sharding via env var for evaluation
 	if shardingEnv, ok := os.LookupEnv("SHARDING_ENABLED"); ok {
-		opts.Sharded, err = strconv.ParseBool(shardingEnv)
+		o.managerOptions.Sharded, err = strconv.ParseBool(shardingEnv)
 		if err != nil {
-			return ctrl.Options{}, fmt.Errorf("error parsing SHARDING_ENABLED env var: %w", err)
+			return fmt.Errorf("error parsing SHARDING_ENABLED env var: %w", err)
 		}
 	}
-	// allow overriding shard ID via env var
-	opts.ShardID = os.Getenv("SHARD_ID")
-	opts.ShardMode = sharding.Mode(os.Getenv("SHARD_MODE"))
 
-	return opts, nil
-}
-
-func setOptionsDefaults(opts ctrl.Options) ctrl.Options {
-	if opts.HealthProbeBindAddress == "" { // "" disables the health server
-		opts.HealthProbeBindAddress = ":8080"
-	}
-
-	// allows us to quickly handover leadership on restarts
-	opts.LeaderElectionReleaseOnCancel = true
-
-	opts.Cache.DefaultTransform = dropUnwantedMetadata
-
-	opts.Controller.RecoverPanic = pointer.Bool(true)
+	// allow overriding shard ID via env var
+	o.managerOptions.ShardID = os.Getenv("SHARD_ID")
+	o.managerOptions.ShardMode = sharding.Mode(os.Getenv("SHARD_MODE"))
 
-	return opts
+	return nil
 }
 
 func dropUnwantedMetadata(i interface{}) (interface{}, error) {

webhosting-operator/config/crd/kustomization.yaml → webhosting-operator/config/manager/crds/kustomization.yaml

@@ -2,8 +2,8 @@
 # since it depends on service name and namespace that are out of this kustomize package.
 # It should be run by config/manager/default
 resources:
-- bases/webhosting.timebertt.dev_websites.yaml
-- bases/webhosting.timebertt.dev_themes.yaml
+- webhosting.timebertt.dev_websites.yaml
+- webhosting.timebertt.dev_themes.yaml
 #+kubebuilder:scaffold:crdkustomizeresource
 
 # the following config is for teaching kustomize how to do kustomization for CRDs.

webhosting-operator/config/manager/default/kustomization.yaml

@@ -20,17 +20,9 @@ images:
   newName: ghcr.io/timebertt/kubernetes-controller-sharding/webhosting-operator
   newTag: latest
 
-generatorOptions:
-  disableNameSuffixHash: true
-
-configMapGenerator:
-- files:
-  - controller_manager_config.yaml
-  name: manager-config
-
 resources:
 - manager.yaml
-- ../../crd
+- ../crds
 - ../rbac
 - auth_proxy_service.yaml
 - auth_proxy_role.yaml

webhosting-operator/config/manager/default/manager.yaml

@@ -21,8 +21,6 @@ spec:
       containers:
       - name: manager
         image: controller:latest
-        args:
-        - "--config=/controller_manager_config.yaml"
         ports:
         - name: metrics
           containerPort: 8080
@@ -48,13 +46,5 @@ spec:
           requests:
             cpu: 250m
             memory: 256Mi
-        volumeMounts:
-        - name: manager-config
-          mountPath: /controller_manager_config.yaml
-          subPath: controller_manager_config.yaml
-      volumes:
-      - name: manager-config
-        configMap:
-          name: manager-config
       serviceAccountName: operator
       terminationGracePeriodSeconds: 30