[ws-manager-mk2] Decide how to apply appamor config based on serverversion to support k8s 1.30+

Tool: gitpod/catfood.gitpod.cloud

Author: geropl

components/ws-manager-mk2/controllers/create.go

@@ -21,6 +21,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/utils/pointer"
 
 	wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
@@ -62,6 +63,7 @@ type startWorkspaceContext struct {
 	IDEPort        int32             `json:"idePort"`
 	SupervisorPort int32             `json:"supervisorPort"`
 	Headless       bool              `json:"headless"`
+	ServerVersion  *version.Info     `json:"serverVersion"`
 }
 
 // createWorkspacePod creates the actual workspace pod based on the definite workspace pod and appropriate
@@ -278,12 +280,13 @@ func createDefiniteWorkspacePod(sctx *startWorkspaceContext) (*corev1.Pod, error
 		"prometheus.io/scrape": "true",
 		"prometheus.io/path":   "/metrics",
 		"prometheus.io/port":   strconv.Itoa(int(sctx.IDEPort)),
-		"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
 		// prevent cluster-autoscaler from removing a node
 		// https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-types-of-pods-can-prevent-ca-from-removing-a-node
 		"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
 	}
 
+	configureAppamor(sctx, annotations, workspaceContainer)
+
 	for k, v := range sctx.Workspace.Annotations {
 		annotations[k] = v
 	}
@@ -689,7 +692,7 @@ func createDefaultSecurityContext() (*corev1.SecurityContext, error) {
 	return res, nil
 }
 
-func newStartWorkspaceContext(ctx context.Context, cfg *config.Configuration, ws *workspacev1.Workspace) (res *startWorkspaceContext, err error) {
+func newStartWorkspaceContext(ctx context.Context, cfg *config.Configuration, ws *workspacev1.Workspace, serverVersion *version.Info) (res *startWorkspaceContext, err error) {
 	// we deliberately do not shadow ctx here as we need the original context later to extract the TraceID
 	span, _ := tracing.FromContext(ctx, "newStartWorkspaceContext")
 	defer tracing.FinishSpan(span, &err)
@@ -711,9 +714,21 @@ func newStartWorkspaceContext(ctx context.Context, cfg *config.Configuration, ws
 		IDEPort:        23000,
 		SupervisorPort: 22999,
 		Headless:       ws.IsHeadless(),
+		ServerVersion:  serverVersion,
 	}, nil
 }
 
+func configureAppamor(sctx *startWorkspaceContext, annotations map[string]string, workspaceContainer *corev1.Container) {
+	// pre K8s 1.30 we need to set the apparmor profile to unconfined as an annotation
+	if sctx.ServerVersion.Major <= "1" && sctx.ServerVersion.Minor < "30" {
+		annotations["container.apparmor.security.beta.kubernetes.io/workspace"] = "unconfined"
+	} else {
+		workspaceContainer.SecurityContext.AppArmorProfile = &corev1.AppArmorProfile{
+			Type: corev1.AppArmorProfileTypeUnconfined,
+		}
+	}
+}
+
 // validCookieChars contains all characters which may occur in an HTTP Cookie value (unicode \u0021 through \u007E),
 // without the characters , ; and / ... I did not find more details about permissible characters in RFC2965, so I took
 // this list of permissible chars from Uncyclopedia.

components/ws-manager-mk2/controllers/suite_test.go

@@ -107,7 +107,7 @@ var _ = BeforeSuite(func() {
 
 	conf := newTestConfig()
 	maintenance := &fakeMaintenance{enabled: false}
-	wsReconciler, err := NewWorkspaceReconciler(k8sManager.GetClient(), k8sManager.GetScheme(), k8sManager.GetEventRecorderFor("workspace"), &conf, metrics.Registry, maintenance)
+	wsReconciler, err := NewWorkspaceReconciler(k8sManager.GetClient(), k8sManager.GetConfig(), k8sManager.GetScheme(), k8sManager.GetEventRecorderFor("workspace"), &conf, metrics.Registry, maintenance)
 	wsMetrics = wsReconciler.metrics
 	Expect(err).ToNot(HaveOccurred())
 	Expect(wsReconciler.SetupWithManager(k8sManager)).To(Succeed())

components/ws-manager-mk2/controllers/workspace_controller.go

@@ -18,6 +18,9 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -47,13 +50,20 @@ const (
 	maintenanceRequeue         = 1 * time.Minute
 )
 
-func NewWorkspaceReconciler(c client.Client, scheme *runtime.Scheme, recorder record.EventRecorder, cfg *config.Configuration, reg prometheus.Registerer, maintenance maintenance.Maintenance) (*WorkspaceReconciler, error) {
+func NewWorkspaceReconciler(c client.Client, restConfig *rest.Config, scheme *runtime.Scheme, recorder record.EventRecorder, cfg *config.Configuration, reg prometheus.Registerer, maintenance maintenance.Maintenance) (*WorkspaceReconciler, error) {
+	// Create kubernetes clientset
+	kubeClient, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create kubernetes client: %w", err)
+	}
+
 	reconciler := &WorkspaceReconciler{
 		Client:      c,
 		Scheme:      scheme,
 		Config:      cfg,
 		maintenance: maintenance,
 		Recorder:    recorder,
+		kubeClient:  kubeClient,
 	}
 
 	metrics, err := newControllerMetrics(reconciler)
@@ -75,6 +85,8 @@ type WorkspaceReconciler struct {
 	metrics     *controllerMetrics
 	maintenance maintenance.Maintenance
 	Recorder    record.EventRecorder
+
+	kubeClient kubernetes.Interface
 }
 
 //+kubebuilder:rbac:groups=workspace.gitpod.io,resources=workspaces,verbs=get;list;watch;create;update;patch;delete
@@ -181,7 +193,8 @@ func (r *WorkspaceReconciler) actOnStatus(ctx context.Context, workspace *worksp
 		// if there isn't a workspace pod and we're not currently deleting this workspace,// create one.
 		switch {
 		case workspace.Status.PodStarts == 0 || workspace.Status.PodStarts-workspace.Status.PodRecreated < 1:
-			sctx, err := newStartWorkspaceContext(ctx, r.Config, workspace)
+			serverVersion := r.getServerVersion(ctx)
+			sctx, err := newStartWorkspaceContext(ctx, r.Config, workspace, serverVersion)
 			if err != nil {
 				log.Error(err, "unable to create startWorkspace context")
 				return ctrl.Result{Requeue: true}, err
@@ -627,6 +640,20 @@ func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
+func (r *WorkspaceReconciler) getServerVersion(ctx context.Context) *version.Info {
+	log := log.FromContext(ctx)
+
+	serverVersion, err := r.kubeClient.Discovery().ServerVersion()
+	if err != nil {
+		log.Error(err, "cannot get server version! Assuming 1.30 going forward")
+		serverVersion = &version.Info{
+			Major: "1",
+			Minor: "30",
+		}
+	}
+	return serverVersion
+}
+
 func SetupIndexer(mgr ctrl.Manager) error {
 	var err error
 	var once sync.Once

components/ws-manager-mk2/main.go

@@ -201,7 +201,7 @@ func main() {
 		<-mgr.Elected()
 
 		workspaceReconciler, err := controllers.NewWorkspaceReconciler(
-			mgr.GetClient(), mgr.GetScheme(), mgr.GetEventRecorderFor("workspace"), &cfg.Manager, metrics.Registry, maintenanceReconciler)
+			mgr.GetClient(), mgr.GetConfig(), mgr.GetScheme(), mgr.GetEventRecorderFor("workspace"), &cfg.Manager, metrics.Registry, maintenanceReconciler)
 		if err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "Workspace")
 			os.Exit(1)