adding vn-agent healthcheck & ability to boot from kconfig

Signed-off-by: Chris Hein <[email protected]>

Author: christopherhein

virtualcluster/cmd/vn-agent/app/options/options.go

@@ -51,6 +51,9 @@ type ServerOption struct {
 	// Port is the vn-agent server listening on.
 	Port uint
 
+	// Kubeconfig is the supercluster Kubeconfig to connect to
+	Kubeconfig string
+
 	// FeatureGates enabled by the user.
 	FeatureGates map[string]bool
 }
@@ -84,6 +87,7 @@ func (o *Options) Flags() cliflag.NamedFlagSets {
 	serverFS.StringVar(&o.CertDirectory, "cert-dir", o.CertDirectory, "CertDirectory is the directory where the TLS certs are located")
 	serverFS.StringVar(&o.TLSCertFile, "tls-cert-file", o.TLSCertFile, "TLSCertFile is the file containing x509 Certificate for HTTPS")
 	serverFS.StringVar(&o.TLSPrivateKeyFile, "tls-private-key-file", o.TLSPrivateKeyFile, "TLSPrivateKeyFile is the file containing x509 private key matching tlsCertFile")
+	serverFS.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
 	serverFS.UintVar(&o.Port, "port", 10550, "Port is the server listening on")
 	serverFS.Var(cliflag.NewMapStringBool(&o.ServerOption.FeatureGates), "feature-gates", "A set of key=value pairs that describe featuregate gates for various features.")
 

virtualcluster/cmd/vn-agent/app/server.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/term"
 	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/healthz"
 
 	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/cmd/vn-agent/app/options"
 	utilflag "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/util/flag"
@@ -90,7 +91,7 @@ func NewVnAgentCommand(stopChan <-chan struct{}) *cobra.Command {
 
 // Run start the vn-agent server.
 func Run(c *config.Config, serverOption *options.ServerOption, stopCh <-chan struct{}) error {
-	handler, err := server.NewServer(c)
+	handler, err := server.NewServer(c, serverOption)
 	if err != nil {
 		return errors.Wrapf(err, "create server")
 	}
@@ -131,6 +132,14 @@ func Run(c *config.Config, serverOption *options.ServerOption, stopCh <-chan str
 		errCh <- err
 	}()
 
+	go func() {
+		// start a health http server.
+		mux := http.NewServeMux()
+		healthz.InstallHandler(mux)
+		klog.Fatal(http.ListenAndServe(":8080", mux))
+		errCh <- err
+	}()
+
 	select {
 	case <-stopCh:
 		klog.Infof("closing server...")

virtualcluster/go.sum

@@ -1082,6 +1082,7 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhD
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
+k8s.io/kubernetes v0.20.2 h1:KxQftTwyg3DRlwKDcI6wla6eLygmMJ21FXlE4o7QcvU=
 k8s.io/kubernetes v0.20.2/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ=

virtualcluster/pkg/controller/controllers/suite_test.go

@@ -44,8 +44,8 @@ var cfg *rest.Config
 var cli client.Client
 var testEnv *envtest.Environment
 
-const timeout = time.Second * 10
-const longTimeout = time.Second * 20
+const timeout = time.Second * 20
+const longTimeout = time.Second * 60
 const interval = time.Millisecond * 250
 
 func TestAPIs(t *testing.T) {

virtualcluster/pkg/vn-agent/server/server.go

@@ -18,14 +18,17 @@ package server
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"net/http"
 	"net/url"
 
 	"github.com/emicklei/go-restful"
 	"github.com/pkg/errors"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	certutil "k8s.io/client-go/util/cert"
 
+	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/cmd/vn-agent/app/options"
 	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/vn-agent/config"
 )
 
@@ -44,7 +47,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 }
 
 // NewServer initializes and configures a vn-agent.Server object to handle HTTP requests.
-func NewServer(cfg *config.Config) (*Server, error) {
+func NewServer(cfg *config.Config, serverOption *options.ServerOption) (*Server, error) {
 	u, err := url.Parse(cfg.KubeletServerHost)
 	if err != nil {
 		return nil, errors.Wrap(err, "parse kubelet server url")
@@ -66,17 +69,27 @@ func NewServer(cfg *config.Config) (*Server, error) {
 			},
 		}
 	} else {
-		restConfig, err := rest.InClusterConfig()
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to get in cluster config")
+		var restConfig *rest.Config
+		var caCrtPool *x509.CertPool
+		if len(serverOption.Kubeconfig) == 0 {
+			restConfig, err = rest.InClusterConfig()
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to get in cluster config")
+			}
+			caCrtPool, err = certutil.NewPool(restConfig.TLSClientConfig.CAFile)
+		} else {
+			// This creates a client, first loading any specified kubeconfig\
+			restConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+				&clientcmd.ClientConfigLoadingRules{ExplicitPath: serverOption.Kubeconfig},
+				&clientcmd.ConfigOverrides{}).ClientConfig()
+			caCrtPool, err = certutil.NewPoolFromBytes(restConfig.TLSClientConfig.CAData)
 		}
 		server.restConfig = restConfig
 		superHttpsUrl, err := url.Parse(restConfig.Host)
 		if err != nil {
 			return nil, errors.Wrapf(err, "unable to parse apiserver address")
 		}
 		server.superAPIServerAddress = superHttpsUrl
-		caCrtPool, err := certutil.NewPool(restConfig.TLSClientConfig.CAFile)
 		if err != nil {
 			return nil, errors.Wrapf(err, "unable to parse ca file")
 		}