Skip to content

Commit 1bacab8

Browse files
WVerlaekWVerlaek
committed
[ws-manager-mk2] Loadgen fixes, concurrent reconciliation
1 parent a9810d6 commit 1bacab8

File tree

31 files changed

+167
-102
lines changed

31 files changed

+167
-102
lines changed

‎components/ws-daemon/pkg/controller/snapshot_controller.go

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"k8s.io/client-go/util/retry"
1212
ctrl "sigs.k8s.io/controller-runtime"
1313
"sigs.k8s.io/controller-runtime/pkg/client"
14+
"sigs.k8s.io/controller-runtime/pkg/controller"
1415
"sigs.k8s.io/controller-runtime/pkg/event"
1516
"sigs.k8s.io/controller-runtime/pkg/log"
1617
"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -21,22 +22,27 @@ import (
2122
// SnapshotReconciler reconciles a Snapshot object
2223
type SnapshotReconciler struct {
2324
client.Client
24-
nodeName string
25-
operations *WorkspaceOperations
25+
maxConcurrentReconciles int
26+
nodeName string
27+
operations *WorkspaceOperations
2628
}
2729

28-
func NewSnapshotController(c client.Client, nodeName string, wso *WorkspaceOperations) *SnapshotReconciler {
30+
func NewSnapshotController(c client.Client, nodeName string, maxConcurrentReconciles int, wso *WorkspaceOperations) *SnapshotReconciler {
2931
return &SnapshotReconciler{
30-
Client: c,
31-
nodeName: nodeName,
32-
operations: wso,
32+
Client: c,
33+
maxConcurrentReconciles: maxConcurrentReconciles,
34+
nodeName: nodeName,
35+
operations: wso,
3336
}
3437
}
3538

3639
// SetupWithManager sets up the controller with the Manager.
3740
func (r *SnapshotReconciler) SetupWithManager(mgr ctrl.Manager) error {
3841
return ctrl.NewControllerManagedBy(mgr).
3942
Named("snapshot").
43+
WithOptions(controller.Options{
44+
MaxConcurrentReconciles: r.maxConcurrentReconciles,
45+
}).
4046
For(&workspacev1.Snapshot{}).
4147
WithEventFilter(snapshotEventFilter(r.nodeName)).
4248
Complete(r)

‎components/ws-daemon/pkg/controller/workspace_controller.go

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"k8s.io/client-go/util/retry"
2727
ctrl "sigs.k8s.io/controller-runtime"
2828
"sigs.k8s.io/controller-runtime/pkg/client"
29+
"sigs.k8s.io/controller-runtime/pkg/controller"
2930
"sigs.k8s.io/controller-runtime/pkg/event"
3031
"sigs.k8s.io/controller-runtime/pkg/log"
3132
"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -49,27 +50,32 @@ type WorkspaceControllerOpts struct {
4950

5051
type WorkspaceController struct {
5152
client.Client
52-
NodeName string
53-
operations *WorkspaceOperations
54-
metrics *workspaceMetrics
53+
NodeName string
54+
maxConcurrentReconciles int
55+
operations *WorkspaceOperations
56+
metrics *workspaceMetrics
5557
}
5658

57-
func NewWorkspaceController(c client.Client, nodeName string, ops *WorkspaceOperations, reg prometheus.Registerer) (*WorkspaceController, error) {
59+
func NewWorkspaceController(c client.Client, nodeName string, maxConcurrentReconciles int, ops *WorkspaceOperations, reg prometheus.Registerer) (*WorkspaceController, error) {
5860
metrics := newWorkspaceMetrics()
5961
reg.Register(metrics)
6062

6163
return &WorkspaceController{
62-
Client: c,
63-
NodeName: nodeName,
64-
operations: ops,
65-
metrics: metrics,
64+
Client: c,
65+
NodeName: nodeName,
66+
maxConcurrentReconciles: maxConcurrentReconciles,
67+
operations: ops,
68+
metrics: metrics,
6669
}, nil
6770
}
6871

6972
// SetupWithManager sets up the controller with the Manager.
7073
func (wsc *WorkspaceController) SetupWithManager(mgr ctrl.Manager) error {
7174
return ctrl.NewControllerManagedBy(mgr).
7275
Named("workspace").
76+
WithOptions(controller.Options{
77+
MaxConcurrentReconciles: wsc.maxConcurrentReconciles,
78+
}).
7379
For(&workspacev1.Workspace{}).
7480
WithEventFilter(eventFilter(wsc.NodeName)).
7581
Complete(wsc)

‎components/ws-daemon/pkg/daemon/config.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@ type Config struct {
3333
}
3434

3535
type WorkspaceControllerConfig struct {
36-
Enabled bool `json:"enabled"`
37-
WorkingAreaSuffix string `json:"workingAreaSuffix"`
36+
Enabled bool `json:"enabled"`
37+
WorkingAreaSuffix string `json:"workingAreaSuffix"`
38+
MaxConcurrentReconciles int `json:maxConcurrentReconciles,omitempty`
3839
}
3940

4041
type RuntimeConfig struct {

‎components/ws-daemon/pkg/daemon/daemon.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ func NewDaemon(config Config) (*Daemon, error) {
207207
return nil, err
208208
}
209209

210-
wsctrl, err := controller.NewWorkspaceController(mgr.GetClient(), nodename, workspaceOps, wrappedReg)
210+
wsctrl, err := controller.NewWorkspaceController(mgr.GetClient(), nodename, config.WorkspaceController.MaxConcurrentReconciles, workspaceOps, wrappedReg)
211211
if err != nil {
212212
return nil, err
213213
}
@@ -216,7 +216,7 @@ func NewDaemon(config Config) (*Daemon, error) {
216216
return nil, err
217217
}
218218

219-
ssctrl := controller.NewSnapshotController(mgr.GetClient(), nodename, workspaceOps)
219+
ssctrl := controller.NewSnapshotController(mgr.GetClient(), nodename, config.WorkspaceController.MaxConcurrentReconciles, workspaceOps)
220220
err = ssctrl.SetupWithManager(mgr)
221221
if err != nil {
222222
return nil, err

‎components/ws-manager-api/go/config/config.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,9 @@ type Configuration struct {
125125
WorkspaceClasses map[string]*WorkspaceClass `json:"workspaceClass"`
126126
// DebugWorkspacePod adds extra finalizer to workspace to prevent it from shutting down. Helps to debug.
127127
DebugWorkspacePod bool `json:"debugWorkspacePod,omitempty"`
128+
// WorkspaceMaxConcurrentReconciles configures the max amount of concurrent workspace reconciliations on
129+
// the workspace controller.
130+
WorkspaceMaxConcurrentReconciles int `json:workspaceMaxConcurrentReconciles,omitempty`
128131
// TimeoutMaxConcurrentReconciles configures the max amount of concurrent workspace reconciliations on
129132
// the timeout controller.
130133
TimeoutMaxConcurrentReconciles int `json:"timeoutMaxConcurrentReconciles,omitempty"`

‎components/ws-manager-mk2/controllers/status.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,10 +188,12 @@ func updateWorkspaceStatus(ctx context.Context, workspace *workspacev1.Workspace
188188
}
189189

190190
func isDisposalFinished(ws *workspacev1.Workspace) bool {
191+
c := wsk8s.GetCondition(ws.Status.Conditions, string(workspacev1.WorkspaceConditionContentReady))
192+
noContentInit := c == nil || c.Status == metav1.ConditionFalse
191193
return wsk8s.ConditionPresentAndTrue(ws.Status.Conditions, string(workspacev1.WorkspaceConditionBackupComplete)) ||
192194
wsk8s.ConditionPresentAndTrue(ws.Status.Conditions, string(workspacev1.WorkspaceConditionBackupFailure)) ||
193195
wsk8s.ConditionPresentAndTrue(ws.Status.Conditions, string(workspacev1.WorkspaceConditionAborted)) ||
194-
wsk8s.ConditionWithStatusAndReason(ws.Status.Conditions, string(workspacev1.WorkspaceConditionContentReady), false, "InitializationFailure")
196+
noContentInit
195197
}
196198

197199
// extractFailure returns a pod failure reason and possibly a phase. If phase is nil then

‎components/ws-manager-mk2/controllers/workspace_controller.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"k8s.io/apimachinery/pkg/runtime"
1616
ctrl "sigs.k8s.io/controller-runtime"
1717
"sigs.k8s.io/controller-runtime/pkg/client"
18+
"sigs.k8s.io/controller-runtime/pkg/controller"
1819
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
1920
"sigs.k8s.io/controller-runtime/pkg/log"
2021

@@ -360,6 +361,9 @@ func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
360361

361362
return ctrl.NewControllerManagedBy(mgr).
362363
Named("workspace").
364+
WithOptions(controller.Options{
365+
MaxConcurrentReconciles: r.Config.WorkspaceMaxConcurrentReconciles,
366+
}).
363367
For(&workspacev1.Workspace{}).
364368
Owns(&corev1.Pod{}).
365369
Complete(r)

‎components/ws-manager-mk2/service/manager.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ func (wsm *WorkspaceManagerServer) StartWorkspace(ctx context.Context, req *wsma
249249
}
250250

251251
var wsr workspacev1.Workspace
252-
err = wait.PollWithContext(ctx, 100*time.Millisecond, 5*time.Second, func(c context.Context) (done bool, err error) {
252+
err = wait.PollWithContext(ctx, 200*time.Millisecond, 15*time.Second, func(c context.Context) (done bool, err error) {
253253
err = wsm.Client.Get(ctx, types.NamespacedName{Namespace: wsm.Config.Namespace, Name: ws.Name}, &wsr)
254254
if err != nil {
255255
return false, nil

‎dev/loadgen/cmd/benchmark.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ var benchmarkCommand = &cobra.Command{
5555
}
5656

5757
var load loadgen.LoadGenerator
58-
load = loadgen.NewFixedLoadGenerator(500*time.Millisecond, 300*time.Millisecond)
58+
load = loadgen.NewFixedLoadGenerator(800*time.Millisecond, 300*time.Millisecond)
5959
load = loadgen.NewWorkspaceCountLimitingGenerator(load, scenario.Workspaces)
6060

6161
template := &api.StartWorkspaceRequest{

‎dev/loadgen/pkg/loadgen/executor.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,9 +222,12 @@ func (w *WsmanExecutor) StopAll(ctx context.Context) error {
222222
if err != nil {
223223
log.Warnf("could not get workspaces: %v", err)
224224
} else {
225-
if len(resp.GetStatus()) == 0 {
225+
n := len(resp.GetStatus())
226+
if n == 0 {
226227
break
227228
}
229+
ex := resp.GetStatus()[0]
230+
log.Infof("%d workspaces remaining, e.g. %s", n, ex.Id)
228231
}
229232

230233
select {

‎install/installer/cmd/testdata/render/agent-smith/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/installer/cmd/testdata/render/aws-setup/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/installer/cmd/testdata/render/custom-pull-repository/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/installer/cmd/testdata/render/customization/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/installer/cmd/testdata/render/external-registry/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)