Skip to content

Commit aba880e

Browse files
ingvagabundingvagabund
committed
UPSTREAM: <carry>: openshift: Machine controller: drain node before machine deletion
The node draining code itself is imported from github.com/openshift/kubernetes-drain. At the same time it's currently impossible to use the controller-runtime client for node draining due to missing Patch operation (kubernetes-sigs/controller-runtime#235). Thus, the machine controller needs to initialize kubeclient as well in order to implement the node draining logic. Once the Patch operation is implemented, the draining logic can be updated to replace kube client with controller runtime client. Also, initialize event recorder to generate node draining event.
1 parent 7bb3fce commit aba880e

File tree

21 files changed

+1127
-30
lines changed

21 files changed

+1127
-30
lines changed

‎Gopkg.lock

Lines changed: 21 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cmd/clusterctl/clusterdeployer/provider/BUILD.bazel

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ go_library(
66
importpath = "github.com/openshift/cluster-api/cmd/clusterctl/clusterdeployer/provider",
77
visibility = ["//visibility:public"],
88
deps = [
9+
"//pkg/apis/cluster/v1alpha1:go_default_library",
910
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
10-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
1111
],
1212
)

‎cmd/clusterctl/cmd/BUILD.bazel

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,6 @@ go_library(
4444
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
4545
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
4646
"//vendor/k8s.io/klog:go_default_library",
47-
"//vendor/sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/clusterclient:go_default_library",
48-
"//vendor/sigs.k8s.io/cluster-api/cmd/clusterctl/phases:go_default_library",
4947
"//vendor/sigs.k8s.io/controller-runtime/pkg/client:go_default_library",
5048
"//vendor/sigs.k8s.io/controller-runtime/pkg/client/config:go_default_library",
5149
"//vendor/sigs.k8s.io/controller-runtime/pkg/manager:go_default_library",

‎cmd/clusterctl/phases/BUILD.bazel

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,13 @@ go_library(
1717
deps = [
1818
"//cmd/clusterctl/clusterdeployer/bootstrap:go_default_library",
1919
"//cmd/clusterctl/clusterdeployer/clusterclient:go_default_library",
20+
"//cmd/clusterctl/clusterdeployer/provider:go_default_library",
2021
"//pkg/apis/cluster/v1alpha1:go_default_library",
22+
"//pkg/util:go_default_library",
2123
"//vendor/github.com/pkg/errors:go_default_library",
2224
"//vendor/k8s.io/api/apps/v1:go_default_library",
2325
"//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
2426
"//vendor/k8s.io/klog:go_default_library",
25-
"//vendor/sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/clusterclient:go_default_library",
26-
"//vendor/sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/provider:go_default_library",
27-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
28-
"//vendor/sigs.k8s.io/cluster-api/pkg/util:go_default_library",
2927
],
3028
)
3129

@@ -34,7 +32,7 @@ go_test(
3432
srcs = ["pivot_test.go"],
3533
embed = [":go_default_library"],
3634
deps = [
35+
"//pkg/apis/cluster/v1alpha1:go_default_library",
3736
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
38-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
3937
],
4038
)

‎cmd/clusterctl/validation/BUILD.bazel

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,14 @@ go_test(
3030
embed = [":go_default_library"],
3131
deps = [
3232
"//pkg/apis:go_default_library",
33+
"//pkg/apis/cluster/common:go_default_library",
34+
"//pkg/apis/cluster/v1alpha1:go_default_library",
35+
"//pkg/apis/cluster/v1alpha1/testutil:go_default_library",
3336
"//vendor/k8s.io/api/core/v1:go_default_library",
3437
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
3538
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
3639
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
3740
"//vendor/k8s.io/client-go/rest:go_default_library",
38-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/common:go_default_library",
39-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
40-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1/testutil:go_default_library",
4141
"//vendor/sigs.k8s.io/controller-runtime/pkg/client:go_default_library",
4242
"//vendor/sigs.k8s.io/controller-runtime/pkg/envtest:go_default_library",
4343
"//vendor/sigs.k8s.io/controller-runtime/pkg/manager:go_default_library",

‎cmd/example-provider/BUILD.bazel

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@ go_library(
66
importpath = "github.com/openshift/cluster-api/cmd/example-provider",
77
visibility = ["//visibility:private"],
88
deps = [
9+
"//pkg/apis:go_default_library",
10+
"//pkg/apis/cluster/common:go_default_library",
11+
"//pkg/client/clientset_generated/clientset:go_default_library",
12+
"//pkg/controller/cluster:go_default_library",
13+
"//pkg/controller/machine:go_default_library",
14+
"//pkg/provider/example/actuators/cluster:go_default_library",
15+
"//pkg/provider/example/actuators/machine:go_default_library",
916
"//vendor/k8s.io/klog:go_default_library",
10-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis:go_default_library",
11-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/common:go_default_library",
12-
"//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset:go_default_library",
13-
"//vendor/sigs.k8s.io/cluster-api/pkg/controller/cluster:go_default_library",
14-
"//vendor/sigs.k8s.io/cluster-api/pkg/controller/machine:go_default_library",
15-
"//vendor/sigs.k8s.io/cluster-api/pkg/provider/example/actuators/cluster:go_default_library",
16-
"//vendor/sigs.k8s.io/cluster-api/pkg/provider/example/actuators/machine:go_default_library",
1717
"//vendor/sigs.k8s.io/controller-runtime/pkg/client/config:go_default_library",
1818
"//vendor/sigs.k8s.io/controller-runtime/pkg/manager:go_default_library",
1919
"//vendor/sigs.k8s.io/controller-runtime/pkg/runtime/signals:go_default_library",

‎pkg/cert/BUILD.bazel

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,4 @@ go_test(
1515
name = "go_default_test",
1616
srcs = ["cert_authority_test.go"],
1717
embed = [":go_default_library"],
18-
deps = ["//vendor/sigs.k8s.io/cluster-api/pkg/cert:go_default_library"],
1918
)

‎pkg/controller/machine/BUILD.bazel

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,15 @@ go_library(
1313
"//pkg/apis/machine/v1beta1:go_default_library",
1414
"//pkg/controller/error:go_default_library",
1515
"//pkg/util:go_default_library",
16+
"//vendor/github.com/go-log/log/info:go_default_library",
17+
"//vendor/github.com/openshift/kubernetes-drain:go_default_library",
1618
"//vendor/k8s.io/api/core/v1:go_default_library",
1719
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
1820
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
1921
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
22+
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
23+
"//vendor/k8s.io/client-go/rest:go_default_library",
24+
"//vendor/k8s.io/client-go/tools/record:go_default_library",
2025
"//vendor/k8s.io/klog:go_default_library",
2126
"//vendor/sigs.k8s.io/controller-runtime/pkg/client:go_default_library",
2227
"//vendor/sigs.k8s.io/controller-runtime/pkg/controller:go_default_library",

‎pkg/controller/machine/controller.go

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,22 @@ package machine
1818

1919
import (
2020
"context"
21+
"fmt"
2122
"os"
23+
"time"
2224

25+
"github.com/go-log/log/info"
2326
machinev1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1"
2427
controllerError "github.com/openshift/cluster-api/pkg/controller/error"
2528
"github.com/openshift/cluster-api/pkg/util"
29+
kubedrain "github.com/openshift/kubernetes-drain"
2630
corev1 "k8s.io/api/core/v1"
2731
apierrors "k8s.io/apimachinery/pkg/api/errors"
2832
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2933
"k8s.io/apimachinery/pkg/runtime"
34+
"k8s.io/client-go/kubernetes"
35+
"k8s.io/client-go/rest"
36+
"k8s.io/client-go/tools/record"
3037
"k8s.io/klog"
3138
"sigs.k8s.io/controller-runtime/pkg/client"
3239
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -38,6 +45,9 @@ import (
3845

3946
const (
4047
NodeNameEnvVar = "NODE_NAME"
48+
49+
// ExcludeNodeDrainingAnnotation annotation explicitly skips node draining if set
50+
ExcludeNodeDrainingAnnotation = "machine.openshift.io/exclude-node-draining"
4151
)
4252

4353
var DefaultActuator Actuator
@@ -49,10 +59,12 @@ func AddWithActuator(mgr manager.Manager, actuator Actuator) error {
4959
// newReconciler returns a new reconcile.Reconciler
5060
func newReconciler(mgr manager.Manager, actuator Actuator) reconcile.Reconciler {
5161
r := &ReconcileMachine{
52-
Client: mgr.GetClient(),
53-
scheme: mgr.GetScheme(),
54-
nodeName: os.Getenv(NodeNameEnvVar),
55-
actuator: actuator,
62+
Client: mgr.GetClient(),
63+
eventRecorder: mgr.GetRecorder("machine-controller"),
64+
config: mgr.GetConfig(),
65+
scheme: mgr.GetScheme(),
66+
nodeName: os.Getenv(NodeNameEnvVar),
67+
actuator: actuator,
5668
}
5769

5870
if r.nodeName == "" {
@@ -80,8 +92,11 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
8092
// ReconcileMachine reconciles a Machine object
8193
type ReconcileMachine struct {
8294
client.Client
95+
config *rest.Config
8396
scheme *runtime.Scheme
8497

98+
eventRecorder record.EventRecorder
99+
85100
actuator Actuator
86101

87102
// nodeName is the name of the node on which the machine controller is running, if not present, it is loaded from NODE_NAME.
@@ -168,6 +183,18 @@ func (r *ReconcileMachine) Reconcile(request reconcile.Request) (reconcile.Resul
168183
}
169184

170185
klog.Infof("Reconciling machine %q triggers delete", name)
186+
187+
// Drain node before deletion
188+
// If a machine is not linked to a node, just delete the machine. Since a node
189+
// can be unlinked from a machine when the node goes NotReady and is removed
190+
// by cloud controller manager. In that case some machines would never get
191+
// deleted without a manual intervention.
192+
if _, exists := m.ObjectMeta.Annotations[ExcludeNodeDrainingAnnotation]; !exists && m.Status.NodeRef != nil {
193+
if err := r.drainNode(m); err != nil {
194+
return reconcile.Result{}, err
195+
}
196+
}
197+
171198
if err := r.actuator.Delete(ctx, cluster, m); err != nil {
172199
if requeueErr, ok := err.(*controllerError.RequeueAfterError); ok {
173200
klog.Infof("Actuator returned requeue-after error: %v", requeueErr)
@@ -233,6 +260,41 @@ func (r *ReconcileMachine) Reconcile(request reconcile.Request) (reconcile.Resul
233260
return reconcile.Result{}, nil
234261
}
235262

263+
func (r *ReconcileMachine) drainNode(machine *machinev1.Machine) error {
264+
kubeClient, err := kubernetes.NewForConfig(r.config)
265+
if err != nil {
266+
return fmt.Errorf("unable to build kube client: %v", err)
267+
}
268+
node, err := kubeClient.CoreV1().Nodes().Get(machine.Status.NodeRef.Name, metav1.GetOptions{})
269+
if err != nil {
270+
return fmt.Errorf("unable to get node %q: %v", machine.Status.NodeRef.Name, err)
271+
}
272+
273+
if err := kubedrain.Drain(
274+
kubeClient,
275+
[]*corev1.Node{node},
276+
&kubedrain.DrainOptions{
277+
Force: true,
278+
IgnoreDaemonsets: true,
279+
DeleteLocalData: true,
280+
GracePeriodSeconds: -1,
281+
Logger: info.New(klog.V(0)),
282+
// If a pod is not evicted in 20 second, retry the eviction next time the
283+
// machine gets reconciled again (to allow other machines to be reconciled)
284+
Timeout: 20 * time.Second,
285+
},
286+
); err != nil {
287+
// Machine still tries to terminate after drain failure
288+
klog.Warningf("drain failed for machine %q: %v", machine.Name, err)
289+
return &controllerError.RequeueAfterError{RequeueAfter: 20 * time.Second}
290+
}
291+
292+
klog.Infof("drain successful for machine %q", machine.Name)
293+
r.eventRecorder.Eventf(machine, corev1.EventTypeNormal, "Deleted", "Node %q drained", node.Name)
294+
295+
return nil
296+
}
297+
236298
func (r *ReconcileMachine) getCluster(ctx context.Context, machine *machinev1.Machine) (*machinev1.Cluster, error) {
237299
if machine.Labels[machinev1.MachineClusterLabelName] == "" {
238300
klog.Infof("Machine %q in namespace %q doesn't specify %q label, assuming nil cluster", machine.Name, machinev1.MachineClusterLabelName, machine.Namespace)

‎pkg/provider/example/actuators/cluster/BUILD.bazel

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ go_library(
66
importpath = "github.com/openshift/cluster-api/pkg/provider/example/actuators/cluster",
77
visibility = ["//visibility:public"],
88
deps = [
9+
"//pkg/apis/cluster/v1alpha1:go_default_library",
10+
"//pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library",
911
"//vendor/k8s.io/api/core/v1:go_default_library",
1012
"//vendor/k8s.io/client-go/tools/record:go_default_library",
11-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
12-
"//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library",
1313
],
1414
)

‎pkg/provider/example/actuators/machine/BUILD.bazel

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ go_library(
66
importpath = "github.com/openshift/cluster-api/pkg/provider/example/actuators/machine",
77
visibility = ["//visibility:public"],
88
deps = [
9+
"//pkg/apis/cluster/v1alpha1:go_default_library",
10+
"//pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library",
911
"//vendor/k8s.io/api/core/v1:go_default_library",
1012
"//vendor/k8s.io/client-go/tools/record:go_default_library",
11-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
12-
"//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library",
1313
],
1414
)

‎test/integration/cluster/BUILD.bazel

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ go_test(
44
name = "go_default_test",
55
srcs = ["cluster_test.go"],
66
deps = [
7+
"//pkg/apis/cluster/v1alpha1:go_default_library",
8+
"//pkg/client/clientset_generated/clientset:go_default_library",
9+
"//pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library",
710
"//vendor/github.com/onsi/ginkgo:go_default_library",
811
"//vendor/github.com/onsi/gomega:go_default_library",
912
"//vendor/k8s.io/api/core/v1:go_default_library",
@@ -12,8 +15,5 @@ go_test(
1215
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
1316
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
1417
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
15-
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
16-
"//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset:go_default_library",
17-
"//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library",
1818
],
1919
)

‎vendor/github.com/go-log/log/BUILD.bazel

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/go-log/log/LICENSE

Lines changed: 19 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)