Merge pull request #281 from cmurphy/list-resources-apiextension

Add API to list resources under namespace tree

Author: k8s-ci-robot

Dockerfile

@@ -14,6 +14,7 @@ COPY internal/ internal/
 
 # Build
 RUN CGO_ENABLED=0 GO111MODULE=on go build -a -o manager ./cmd/manager/main.go
+RUN CGO_ENABLED=0 GO111MODULE=on go build -a -o apiextension ./cmd/apiextension/main.go
 
 # Copied from kubebuilder scaffold to run as nonroot at
 # https://github.com/kubernetes-sigs/kubebuilder/blob/7af89cb00c224c57ece37dc14ea37caf1eb769db/pkg/scaffold/v2/dockerfile.go#L60
@@ -22,5 +23,6 @@ RUN CGO_ENABLED=0 GO111MODULE=on go build -a -o manager ./cmd/manager/main.go
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/apiextension .
 USER nonroot:nonroot
 ENTRYPOINT ["/manager"]

Makefile

@@ -113,6 +113,7 @@ test-only: build-setup-envtest
 # Builds all binaries (manager and kubectl) and manifests
 build: generate fmt vet staticcheck manifests
 	go build -o bin/manager ./cmd/manager/main.go
+	go build -o bin/apiextension ./cmd/apiextension/main.go
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
 	     -o bin/kubectl/kubectl-hns_linux_amd64 \
 	     -ldflags="-X sigs.k8s.io/hierarchical-namespaces/internal/version.Version=${HNC_IMG_TAG}" \

api/v1alpha2/groupversion_info.go

@@ -24,10 +24,13 @@ import (
 )
 
 var (
-	// GroupVersion is group version used to register these objects
+	// GroupVersion is group version used to register the main HNC objects.
 	GroupVersion = schema.GroupVersion{Group: "hnc.x-k8s.io", Version: "v1alpha2"}
 
-	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	// ResourcesGroupVersion is the group used to register resources in an HNC tree.
+	ResourcesGroupVersion = schema.GroupVersion{Group: "resources.hnc.x-k8s.io", Version: "v1alpha2"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme.
 	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
 
 	// AddToScheme adds the types in this group-version to the given scheme.

cmd/apiextension/main.go

@@ -0,0 +1,210 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/gorilla/mux"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/discovery/cached/memory"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/dynamic/dynamicinformer"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	corecache "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/clientcmd"
+	apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	api "sigs.k8s.io/hierarchical-namespaces/api/v1alpha2"
+	"sigs.k8s.io/hierarchical-namespaces/internal/apiextension/apiresources"
+	"sigs.k8s.io/hierarchical-namespaces/internal/apiextension/clients"
+	"sigs.k8s.io/hierarchical-namespaces/internal/apiextension/handlers"
+)
+
+const (
+	kubeSystemNamespace = "kube-system"
+	extensionConfigMap  = "extension-apiserver-authentication"
+	clientCAKey         = "requestheader-client-ca-file"
+)
+
+var (
+	setupLog      = zap.New().WithName("setup-apiext")
+	listenAddress string
+	certPath      string
+	keyPath       string
+	debug         bool
+)
+
+func main() {
+	parseFlags()
+	cfg, err := getConfig()
+	if err != nil {
+		setupLog.Error(err, "unable to get cluster config")
+		os.Exit(1)
+	}
+	server(context.Background(), cfg)
+}
+
+func parseFlags() {
+	setupLog.Info("Parsing flags")
+	flag.StringVar(&listenAddress, "address", ":7443", "The address to listen on.")
+	flag.StringVar(&certPath, "cert", "", "Path to the server cert.")
+	flag.StringVar(&keyPath, "key", "", "Path to the server key.")
+	flag.BoolVar(&debug, "debug", false, "Enable debug logging.")
+	flag.Parse()
+}
+
+func getConfig() (*rest.Config, error) {
+	cfg, err := rest.InClusterConfig()
+	if err == nil {
+		return cfg, nil
+	}
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if kubeconfig == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return nil, fmt.Errorf("could not get kubeconfig: %w", err)
+		}
+		kubeconfig = filepath.Join(home, ".kube", "config")
+	}
+	cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func getMapper(cfg *rest.Config) (meta.RESTMapper, error) {
+	k8sClient, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	// rest mapper setup inspired by https://github.com/rancher/wrangler/blob/3032665ca5611788334c8e49516014278160ebe2/pkg/clients/clients.go#L134-L135
+	// which is in turn borrowed from https://ymmt2005.hatenablog.com/entry/2020/04/14/An_example_of_using_dynamic_client_of_k8s.io/client-go
+	cache := memory.NewMemCacheClient(k8sClient.Discovery())
+	return restmapper.NewDeferredDiscoveryRESTMapper(cache), nil
+}
+
+func server(ctx context.Context, cfg *rest.Config) {
+	discovery, err := discovery.NewDiscoveryClientForConfig(cfg)
+	if err != nil {
+		setupLog.Error(err, "could not start watcher")
+		os.Exit(1)
+	}
+	clientGetter := clients.MediaTypeClientGetter(cfg)
+	dynamicFactory, err := getDynamicInformerFactory(cfg)
+	if err != nil {
+		setupLog.Error(err, "could not get dynammic client for config")
+		os.Exit(1)
+	}
+	crdInformer, apiServiceInformer := setUpAPIInformers(dynamicFactory, ctx.Done())
+	mapper, err := getMapper(cfg)
+	if err != nil {
+		setupLog.Error(err, "could not get REST mapper for config")
+		os.Exit(1)
+	}
+	apis := apiresources.WatchAPIResources(ctx, discovery, crdInformer, apiServiceInformer, mapper)
+	factory, err := getInformerFactory(cfg)
+	if err != nil {
+		setupLog.Error(err, "could not get informer factory")
+		os.Exit(1)
+	}
+	namespaceCache, configMapCache, err := setUpInformers(factory, ctx.Done())
+	if err != nil {
+		setupLog.Error(err, "failed to set up informers")
+		os.Exit(1)
+	}
+	mux := mux.NewRouter()
+	pathPrefix := fmt.Sprintf("/apis/%s", api.ResourcesGroupVersion.String())
+	mux.HandleFunc(pathPrefix, handlers.DiscoveryHandler(apis))
+	mux.HandleFunc(fmt.Sprintf("%s/{resource}", pathPrefix), handlers.Forwarder(clientGetter, apis))
+	mux.HandleFunc(fmt.Sprintf("%s/namespaces/{namespace}/{resource}", pathPrefix), handlers.NamespaceHandler(clientGetter, apis, namespaceCache))
+	mux.Use(handlers.AuthenticateMiddleware(configMapCache, extensionConfigMap))
+
+	clientCA, err := getClientCA(configMapCache)
+	if err != nil {
+		setupLog.Error(err, "could not get client CA from configmap")
+		os.Exit(1)
+	}
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM([]byte(clientCA))
+	tlsConfig := &tls.Config{
+		ClientCAs:  caCertPool,
+		ClientAuth: tls.RequireAndVerifyClientCert,
+	}
+	server := http.Server{
+		Addr:      listenAddress,
+		Handler:   mux,
+		TLSConfig: tlsConfig,
+	}
+	setupLog.Info(fmt.Sprintf("starting server on %s", listenAddress))
+	err = server.ListenAndServeTLS(certPath, keyPath)
+	if err != nil {
+		setupLog.Error(err, "could not start server")
+		os.Exit(1)
+	}
+}
+
+func getClientCA(configMapCache corecache.ConfigMapNamespaceLister) (string, error) {
+	config, err := configMapCache.Get(extensionConfigMap)
+	if err != nil {
+		return "", err
+	}
+	clientCA, ok := config.Data[clientCAKey]
+	if !ok {
+		return "", fmt.Errorf("invalid extension config")
+	}
+	return string(clientCA), nil
+}
+
+func getInformerFactory(cfg *rest.Config) (informers.SharedInformerFactory, error) {
+	clientset, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return informers.NewSharedInformerFactory(clientset, 0), nil
+}
+
+func getDynamicInformerFactory(cfg *rest.Config) (dynamicinformer.DynamicSharedInformerFactory, error) {
+	dynamicClient, err := dynamic.NewForConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, time.Minute), nil
+}
+
+// setUpInformers returns listers (i.e. cache interfaces) for namespaces and for configmaps in the kube-system namespace.
+// See https://medium.com/codex/explore-client-go-informer-patterns-4415bb5f1fbd for information on the informer factory pattern.
+func setUpInformers(factory informers.SharedInformerFactory, stop <-chan struct{}) (corecache.NamespaceLister, corecache.ConfigMapNamespaceLister, error) {
+	namespaceInformer := factory.Core().V1().Namespaces()
+	configMapInformer := factory.Core().V1().ConfigMaps()
+	go factory.Start(stop)
+	if !cache.WaitForCacheSync(stop, namespaceInformer.Informer().HasSynced, configMapInformer.Informer().HasSynced) {
+		return nil, nil, fmt.Errorf("cached failed to sync")
+	}
+	return namespaceInformer.Lister(), configMapInformer.Lister().ConfigMaps(kubeSystemNamespace), nil
+}
+
+// setUpAPIInformers returns informer objects for CRDs and APIServices, which are used for discoverying resources.
+// These informer objects can be used later to add event handlers for when CRDs or APIServies are added, modified, or removed.
+func setUpAPIInformers(factory dynamicinformer.DynamicSharedInformerFactory, stop <-chan struct{}) (cache.SharedIndexInformer, cache.SharedIndexInformer) {
+	crdGVR := apiextv1.SchemeGroupVersion.WithResource("customresourcedefinitions")
+	crdInformer := factory.ForResource(crdGVR).Informer()
+	apiServiceGVR := apiregv1.SchemeGroupVersion.WithResource("apiservices")
+	apiServiceInformer := factory.ForResource(apiServiceGVR).Informer()
+	go factory.Start(stop)
+	return crdInformer, apiServiceInformer
+}

config/apiextension/apiextension.yaml

@@ -0,0 +1,51 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: resourcelist-apiextension
+  name: resourcelist-apiextension
+  namespace: system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: resourcelist-apiextension
+  template:
+    metadata:
+      labels:
+        app: resourcelist-apiextension
+    spec:
+      securityContext:
+        fsGroup: 2000
+        runAsNonRoot: true
+        runAsUser: 1000
+      containers:
+      - image: controller:latest # this is usually overridden by kustomize
+        name: resourcelist
+        command:
+        - /apiextension
+        args:
+        - "--cert=/certs/tls.crt"
+        - "--key=/certs/tls.key"
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+        - name: certs
+          mountPath: /certs
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            drop: ["ALL"]
+        ports:
+        - containerPort: 7443
+          name: server
+          protocol: TCP
+      volumes:
+      - secret:
+          defaultMode: 420
+          secretName: hnc-resourcelist-apiextension
+        name: certs

config/apiextension/apiservice.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1alpha2.resources.hnc.x-k8s.io
+spec:
+  group: resources.hnc.x-k8s.io
+  version: v1alpha2
+  groupPriorityMinimum: 10
+  versionPriority: 10
+  service:
+    namespace: hnc-system
+    name: resourcelist
+    port: 7443

config/apiextension/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+- apiextension.yaml
+- service.yaml
+- apiservice.yaml

config/apiextension/service.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: resourcelist
+  name: resourcelist
+spec:
+  ports:
+  - port: 7443
+    protocol: TCP
+    targetPort: 7443
+  selector:
+    app: resourcelist

config/certmanager/certificate.yaml

@@ -23,3 +23,16 @@ spec:
     kind: Issuer
     name: selfsigned-issuer
   secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: resourcelist
+spec:
+  dnsNames:
+  - $(APIEXT_SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local
+  - $(APIEXT_SERVICE_NAME).$(SERVICE_NAMESPACE).svc
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: hnc-resourcelist

config/internalcert/manifests.yaml

@@ -1,5 +1,12 @@
+---
 apiVersion: v1
 kind: Secret
 metadata:
   name: webhook-server-cert
   namespace: system
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: resourcelist
+  namespace: system

config/rbac/kustomization.yaml

@@ -6,3 +6,5 @@ resources:
 - aggregate_to_admin.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
+- resourcelist_role.yaml
+- resourcelist_rolebinding.yaml

config/rbac/resourcelist_role.yaml

@@ -0,0 +1,15 @@
+# Allow the resourcelist-apiextension deployment to read resources in order to re-register them as HNC resources.
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: resourcelist
+rules:
+- apiGroups:
+  - "*"
+  resources:
+  - "*"
+  verbs:
+  - get
+  - list
+  - watch