Implement the NestedAPIServer controller

PROJECT

@@ -12,4 +12,9 @@ resources:
   group: controlplane
   kind: NestedEtcd
   version: v1alpha4
+- api:
+    crdVersion: v1
+  group: controlplane
+  kind: NestedAPIServer
+  version: v1alpha4
 version: 3-alpha

apis/controlplane/v1alpha4/nestedapiserver_types.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha4
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1"
+)
+
+// NestedAPIServerSpec defines the desired state of NestedAPIServer
+type NestedAPIServerSpec struct {
+	// NestedComponentSpec contains the common and user-specified information that are
+	// required for creating the component
+	// +optional
+	NestedComponentSpec `json:",inline"`
+}
+
+// NestedAPIServerStatus defines the observed state of NestedAPIServer
+type NestedAPIServerStatus struct {
+	// APIServerService is the reference to the service that expose the APIServer
+	// +optional
+	APIServerService *corev1.ObjectReference `json:"apiserverService,omitempty"`
+
+	// CommonStatus allows addons status monitoring
+	addonv1alpha1.CommonStatus `json:",inline"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:resource:scope=Namespaced,path=nestedapiservers,shortName=napiserver
+//+kubebuilder:categories=capi,capn
+//+kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase"
+//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+//+kubebuilder:subresource:status
+
+// NestedAPIServer is the Schema for the nestedapiservers API
+type NestedAPIServer struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   NestedAPIServerSpec   `json:"spec,omitempty"`
+	Status NestedAPIServerStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// NestedAPIServerList contains a list of NestedAPIServer
+type NestedAPIServerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []NestedAPIServer `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&NestedAPIServer{}, &NestedAPIServerList{})
+}

apis/controlplane/v1alpha4/nestedcomponentspec_types.go → apis/controlplane/v1alpha4/nestedcomponent_types.go

@@ -21,6 +21,7 @@ import (
 	addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1"
 )
 
+// NestedComponentSpec defines the common fields for nested components
 type NestedComponentSpec struct {
 	// NestedComponentSpec defines the common information for creating the
 	// component
@@ -40,3 +41,18 @@ type NestedComponentSpec struct {
 	// +optional
 	Replicas int32 `json:"replicas,omitempty"`
 }
+
+type ComponentPhase string
+
+const (
+	Ready   ComponentPhase = "Ready"
+	Unready ComponentPhase = "Unready"
+)
+
+type ComponentKind string
+
+const (
+	APIServer         ComponentKind = "NestedAPIServer"
+	Etcd              ComponentKind = "NestedEtcd"
+	ControllerManager ComponentKind = "NestedControllerManager"
+)

apis/controlplane/v1alpha4/nestedetcd_types.go

@@ -21,13 +21,6 @@ import (
 	addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1"
 )
 
-type NestedEtcdPhase string
-
-const (
-	NestedEtcdReady   NestedEtcdPhase = "ready"
-	NestedEtcdUnready NestedEtcdPhase = "unready"
-)
-
 // NestedEtcdSpec defines the desired state of NestedEtcd
 type NestedEtcdSpec struct {
 	// NestedComponentSpec contains the common and user-specified information

config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{.nestedAPIServerName}}
+  namespace: {{.nestedAPIServerNamespace}}
+  labels: 
+    component-name: {{.nestedAPIServerName}}
+spec:
+  selector:
+    component-name: {{.nestedAPIServerName}}
+  type: NodePort
+  ports:
+  - port: 6443
+    protocol: TCP
+    targetPort: api

config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml

@@ -0,0 +1,99 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{.nestedAPIServerName}}
+  namespace: {{.nestedAPIServerNamespace}} 
+spec:
+  revisionHistoryLimit: 10
+  serviceName: {{.nestedAPIServerName}}
+  selector:
+    matchLabels:
+      component-name: apiserver
+  # apiserver will not be updated, unless it is deleted
+  updateStrategy:
+    type: OnDelete
+  template:
+    metadata:
+      labels:
+        component-name: {{.nestedAPIServerName}}
+    spec:
+      hostname: apiserver
+      subdomain: apiserver-svc
+      containers:
+      - name: {{.nestedAPIServerName}}
+        image: virtualcluster/apiserver-v1.16.2
+        imagePullPolicy: Always
+        command:
+        - kube-apiserver
+        args:
+        - --bind-address=0.0.0.0
+        - --allow-privileged=true
+        - --anonymous-auth=true
+        - --client-ca-file=/etc/kubernetes/pki/root/tls.crt
+        - --tls-cert-file=/etc/kubernetes/pki/apiserver/tls.crt
+        - --tls-private-key-file=/etc/kubernetes/pki/apiserver/tls.key
+        - --kubelet-https=true
+        - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver/tls.crt
+        - --kubelet-client-key=/etc/kubernetes/pki/apiserver/tls.key
+        - --enable-bootstrap-token-auth=true
+        - --etcd-servers=https://{{.nestedEtcdName}}-0.{{.nestedEtcdName}}:2379
+        - --etcd-cafile=/etc/kubernetes/pki/root/tls.crt
+        - --etcd-certfile=/etc/kubernetes/pki/apiserver/tls.crt
+        - --etcd-keyfile=/etc/kubernetes/pki/apiserver/tls.key
+        - --service-account-key-file=/etc/kubernetes/pki/service-account/tls.key
+        - --service-cluster-ip-range=10.32.0.0/16
+        - --service-node-port-range=30000-32767
+        - --authorization-mode=Node,RBAC
+        - --runtime-config=api/all
+        - --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
+        - --apiserver-count=1
+        - --endpoint-reconciler-type=master-count
+        - --v=2
+        ports:
+        - containerPort: 6443
+          protocol: TCP
+          name: api
+        livenessProbe:
+          # since we set anonymous-auth to false, we use tcp instead of https
+          tcpSocket:
+            port: 6443
+          failureThreshold: 8
+          initialDelaySeconds: 15
+          periodSeconds: 10
+          timeoutSeconds: 15
+        readinessProbe:
+          httpGet:
+            port: 6443
+            path: /healthz
+            scheme: HTTPS
+          failureThreshold: 8
+          initialDelaySeconds: 5
+          periodSeconds: 2
+          timeoutSeconds: 30
+        volumeMounts:
+        - mountPath: /etc/kubernetes/pki/apiserver
+          name: {{.nestedControlPlaneName}}-apiserver-client-crt
+          readOnly: true
+        - mountPath: /etc/kubernetes/pki/root
+          name: {{.nestedControlPlaneName}}-apiserver
+          readOnly: true
+        - mountPath: /etc/kubernetes/pki/service-account
+          name: {{.nestedControlPlaneName}}-sa
+          readOnly: true
+      terminationGracePeriodSeconds: 30
+      dnsConfig:
+        searches:
+        - cluster.local
+      volumes:
+      - name: {{.nestedControlPlaneName}}-apiserver-client-crt
+        secret:
+          defaultMode: 420
+          secretName: {{.nestedControlPlaneName}}-apiserver-client-crt
+      - name: {{.nestedControlPlaneName}}-apiserver
+        secret:
+          defaultMode: 420
+          secretName: {{.nestedControlPlaneName}}-apiserver
+      - name: {{.nestedControlPlaneName}}-sa
+        secret:
+          defaultMode: 420
+          secretName: {{.nestedControlPlaneName}}-sa

config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml

@@ -1,26 +1,25 @@
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: {{.nestedetcdName}}
-  namespace: {{.nestedetcdNamespace}}
+  name: {{.nestedEtcdName}}
+  namespace: {{.nestedEtcdNamespace}}
 spec:
-  replicas: {{.nestedetcdStsReplicas}}
   revisionHistoryLimit: 10
-  serviceName: {{.nestedetcdName}}
+  serviceName: {{.nestedEtcdName}}
   selector:
     matchLabels:
-      component-name: {{.nestedetcdName}}
+      component-name: {{.nestedEtcdName}}
   # etcd will not be updated, unless it is deleted
   updateStrategy:
     type: OnDelete
   template:
     metadata:
       labels:
-        component-name: {{.nestedetcdName}}
+        component-name: {{.nestedEtcdName}}
     spec:
       subdomain: etcd
       containers:
-      - name: {{.nestedetcdName}}
+      - name: {{.nestedEtcdName}}
         image: virtualcluster/etcd-v3.4.0
         imagePullPolicy: Always
         command:
@@ -43,9 +42,9 @@ spec:
         - --peer-key-file=/etc/kubernetes/pki/etcd/tls.key
         - --listen-peer-urls=https://0.0.0.0:2380
         - --listen-client-urls=https://0.0.0.0:2379
-        - --initial-advertise-peer-urls=https://$(HOSTNAME).{{.nestedetcdName}}:2380
+        - --initial-advertise-peer-urls=https://$(HOSTNAME).{{.nestedEtcdName}}:2380
         # we use a headless service to encapsulate each pod
-        - --advertise-client-urls=https://$(HOSTNAME).{{.nestedetcdName}}:2379
+        - --advertise-client-urls=https://$(HOSTNAME).{{.nestedEtcdName}}:2379
         - --initial-cluster-state=new
         - --initial-cluster-token=vc-etcd
         - --data-dir=/var/lib/etcd/data