[node-labeler] Do not return an error if the node does not exist (#16832)

* [node-labeler] Do not return an error if the node does not exist

* Increase maxConcurrentReconciles to 20

Author: aledbf

components/node-labeler/cmd/run.go

@@ -72,7 +72,10 @@ var runCmd = &cobra.Command{
 			client,
 		}
 
-		c, err := controller.New("pod-watcher", mgr, controller.Options{Reconciler: r})
+		c, err := controller.New("pod-watcher", mgr, controller.Options{
+			Reconciler:              r,
+			MaxConcurrentReconciles: 20,
+		})
 		if err != nil {
 			log.WithError(err).Fatal("unable to bind controller watch event handler")
 		}
@@ -188,6 +191,12 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r
 		time.Sleep(1 * time.Second)
 		err := updateLabel(labelToUpdate, false, nodeName, r)
 		if err != nil {
+			// this is a edge case when cluster-autoscaler removes a node
+			// (all the running pods will be removed after that)
+			if errors.IsNotFound(err) {
+				return reconcile.Result{}, nil
+			}
+
 			log.WithError(err).Error("unexpected error removing node label")
 			return reconcile.Result{RequeueAfter: time.Second * 10}, err
 		}