Skip to content

Commit adeb3d3

Browse files
WVerlaekWVerlaek
committed
[ws-manager-mk2] Loadgen fixes, concurrent reconciliation
1 parent 0fa0927 commit adeb3d3

File tree

31 files changed

+167
-102
lines changed

31 files changed

+167
-102
lines changed

‎components/ws-daemon/pkg/controller/snapshot_controller.go

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"k8s.io/client-go/util/retry"
1212
ctrl "sigs.k8s.io/controller-runtime"
1313
"sigs.k8s.io/controller-runtime/pkg/client"
14+
"sigs.k8s.io/controller-runtime/pkg/controller"
1415
"sigs.k8s.io/controller-runtime/pkg/event"
1516
"sigs.k8s.io/controller-runtime/pkg/log"
1617
"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -21,22 +22,27 @@ import (
2122
// SnapshotReconciler reconciles a Snapshot object
2223
type SnapshotReconciler struct {
2324
client.Client
24-
nodeName string
25-
operations *WorkspaceOperations
25+
maxConcurrentReconciles int
26+
nodeName string
27+
operations *WorkspaceOperations
2628
}
2729

28-
func NewSnapshotController(c client.Client, nodeName string, wso *WorkspaceOperations) *SnapshotReconciler {
30+
func NewSnapshotController(c client.Client, nodeName string, maxConcurrentReconciles int, wso *WorkspaceOperations) *SnapshotReconciler {
2931
return &SnapshotReconciler{
30-
Client: c,
31-
nodeName: nodeName,
32-
operations: wso,
32+
Client: c,
33+
maxConcurrentReconciles: maxConcurrentReconciles,
34+
nodeName: nodeName,
35+
operations: wso,
3336
}
3437
}
3538

3639
// SetupWithManager sets up the controller with the Manager.
3740
func (r *SnapshotReconciler) SetupWithManager(mgr ctrl.Manager) error {
3841
return ctrl.NewControllerManagedBy(mgr).
3942
Named("snapshot").
43+
WithOptions(controller.Options{
44+
MaxConcurrentReconciles: r.maxConcurrentReconciles,
45+
}).
4046
For(&workspacev1.Snapshot{}).
4147
WithEventFilter(snapshotEventFilter(r.nodeName)).
4248
Complete(r)

‎components/ws-daemon/pkg/controller/workspace_controller.go

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"k8s.io/client-go/util/retry"
2727
ctrl "sigs.k8s.io/controller-runtime"
2828
"sigs.k8s.io/controller-runtime/pkg/client"
29+
"sigs.k8s.io/controller-runtime/pkg/controller"
2930
"sigs.k8s.io/controller-runtime/pkg/event"
3031
"sigs.k8s.io/controller-runtime/pkg/log"
3132
"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -49,27 +50,32 @@ type WorkspaceControllerOpts struct {
4950

5051
type WorkspaceController struct {
5152
client.Client
52-
NodeName string
53-
operations *WorkspaceOperations
54-
metrics *workspaceMetrics
53+
NodeName string
54+
maxConcurrentReconciles int
55+
operations *WorkspaceOperations
56+
metrics *workspaceMetrics
5557
}
5658

57-
func NewWorkspaceController(c client.Client, nodeName string, ops *WorkspaceOperations, reg prometheus.Registerer) (*WorkspaceController, error) {
59+
func NewWorkspaceController(c client.Client, nodeName string, maxConcurrentReconciles int, ops *WorkspaceOperations, reg prometheus.Registerer) (*WorkspaceController, error) {
5860
metrics := newWorkspaceMetrics()
5961
reg.Register(metrics)
6062

6163
return &WorkspaceController{
62-
Client: c,
63-
NodeName: nodeName,
64-
operations: ops,
65-
metrics: metrics,
64+
Client: c,
65+
NodeName: nodeName,
66+
maxConcurrentReconciles: maxConcurrentReconciles,
67+
operations: ops,
68+
metrics: metrics,
6669
}, nil
6770
}
6871

6972
// SetupWithManager sets up the controller with the Manager.
7073
func (wsc *WorkspaceController) SetupWithManager(mgr ctrl.Manager) error {
7174
return ctrl.NewControllerManagedBy(mgr).
7275
Named("workspace").
76+
WithOptions(controller.Options{
77+
MaxConcurrentReconciles: wsc.maxConcurrentReconciles,
78+
}).
7379
For(&workspacev1.Workspace{}).
7480
WithEventFilter(eventFilter(wsc.NodeName)).
7581
Complete(wsc)

‎components/ws-daemon/pkg/daemon/config.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,9 @@ type Config struct {
3535
}
3636

3737
type WorkspaceControllerConfig struct {
38-
Enabled bool `json:"enabled"`
39-
WorkingAreaSuffix string `json:"workingAreaSuffix"`
38+
Enabled bool `json:"enabled"`
39+
WorkingAreaSuffix string `json:"workingAreaSuffix"`
40+
MaxConcurrentReconciles int `json:maxConcurrentReconciles,omitempty`
4041
}
4142

4243
type RuntimeConfig struct {

‎components/ws-daemon/pkg/daemon/daemon.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ func NewDaemon(config Config) (*Daemon, error) {
208208
return nil, err
209209
}
210210

211-
wsctrl, err := controller.NewWorkspaceController(mgr.GetClient(), nodename, workspaceOps, wrappedReg)
211+
wsctrl, err := controller.NewWorkspaceController(mgr.GetClient(), nodename, config.WorkspaceController.MaxConcurrentReconciles, workspaceOps, wrappedReg)
212212
if err != nil {
213213
return nil, err
214214
}
@@ -217,7 +217,7 @@ func NewDaemon(config Config) (*Daemon, error) {
217217
return nil, err
218218
}
219219

220-
ssctrl := controller.NewSnapshotController(mgr.GetClient(), nodename, workspaceOps)
220+
ssctrl := controller.NewSnapshotController(mgr.GetClient(), nodename, config.WorkspaceController.MaxConcurrentReconciles, workspaceOps)
221221
err = ssctrl.SetupWithManager(mgr)
222222
if err != nil {
223223
return nil, err

‎components/ws-manager-api/go/config/config.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,9 @@ type Configuration struct {
125125
WorkspaceClasses map[string]*WorkspaceClass `json:"workspaceClass"`
126126
// DebugWorkspacePod adds extra finalizer to workspace to prevent it from shutting down. Helps to debug.
127127
DebugWorkspacePod bool `json:"debugWorkspacePod,omitempty"`
128+
// WorkspaceMaxConcurrentReconciles configures the max amount of concurrent workspace reconciliations on
129+
// the workspace controller.
130+
WorkspaceMaxConcurrentReconciles int `json:workspaceMaxConcurrentReconciles,omitempty`
128131
// TimeoutMaxConcurrentReconciles configures the max amount of concurrent workspace reconciliations on
129132
// the timeout controller.
130133
TimeoutMaxConcurrentReconciles int `json:"timeoutMaxConcurrentReconciles,omitempty"`

‎components/ws-manager-mk2/controllers/status.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,10 +188,12 @@ func updateWorkspaceStatus(ctx context.Context, workspace *workspacev1.Workspace
188188
}
189189

190190
func isDisposalFinished(ws *workspacev1.Workspace) bool {
191+
c := wsk8s.GetCondition(ws.Status.Conditions, string(workspacev1.WorkspaceConditionContentReady))
192+
noContentInit := c == nil || c.Status == metav1.ConditionFalse
191193
return wsk8s.ConditionPresentAndTrue(ws.Status.Conditions, string(workspacev1.WorkspaceConditionBackupComplete)) ||
192194
wsk8s.ConditionPresentAndTrue(ws.Status.Conditions, string(workspacev1.WorkspaceConditionBackupFailure)) ||
193195
wsk8s.ConditionPresentAndTrue(ws.Status.Conditions, string(workspacev1.WorkspaceConditionAborted)) ||
194-
wsk8s.ConditionWithStatusAndReason(ws.Status.Conditions, string(workspacev1.WorkspaceConditionContentReady), false, "InitializationFailure")
196+
noContentInit
195197
}
196198

197199
// extractFailure returns a pod failure reason and possibly a phase. If phase is nil then

‎components/ws-manager-mk2/controllers/workspace_controller.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"k8s.io/apimachinery/pkg/runtime"
1616
ctrl "sigs.k8s.io/controller-runtime"
1717
"sigs.k8s.io/controller-runtime/pkg/client"
18+
"sigs.k8s.io/controller-runtime/pkg/controller"
1819
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
1920
"sigs.k8s.io/controller-runtime/pkg/log"
2021

@@ -338,6 +339,9 @@ func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
338339

339340
return ctrl.NewControllerManagedBy(mgr).
340341
Named("workspace").
342+
WithOptions(controller.Options{
343+
MaxConcurrentReconciles: r.Config.WorkspaceMaxConcurrentReconciles,
344+
}).
341345
For(&workspacev1.Workspace{}).
342346
Owns(&corev1.Pod{}).
343347
Complete(r)

‎components/ws-manager-mk2/service/manager.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ func (wsm *WorkspaceManagerServer) StartWorkspace(ctx context.Context, req *wsma
241241
}
242242

243243
var wsr workspacev1.Workspace
244-
err = wait.PollWithContext(ctx, 100*time.Millisecond, 5*time.Second, func(c context.Context) (done bool, err error) {
244+
err = wait.PollWithContext(ctx, 200*time.Millisecond, 15*time.Second, func(c context.Context) (done bool, err error) {
245245
err = wsm.Client.Get(ctx, types.NamespacedName{Namespace: wsm.Config.Namespace, Name: ws.Name}, &wsr)
246246
if err != nil {
247247
return false, nil

‎dev/loadgen/cmd/benchmark.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ var benchmarkCommand = &cobra.Command{
5555
}
5656

5757
var load loadgen.LoadGenerator
58-
load = loadgen.NewFixedLoadGenerator(500*time.Millisecond, 300*time.Millisecond)
58+
load = loadgen.NewFixedLoadGenerator(800*time.Millisecond, 300*time.Millisecond)
5959
load = loadgen.NewWorkspaceCountLimitingGenerator(load, scenario.Workspaces)
6060

6161
template := &api.StartWorkspaceRequest{

‎dev/loadgen/pkg/loadgen/executor.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,9 +222,12 @@ func (w *WsmanExecutor) StopAll(ctx context.Context) error {
222222
if err != nil {
223223
log.Warnf("could not get workspaces: %v", err)
224224
} else {
225-
if len(resp.GetStatus()) == 0 {
225+
n := len(resp.GetStatus())
226+
if n == 0 {
226227
break
227228
}
229+
ex := resp.GetStatus()[0]
230+
log.Infof("%d workspaces remaining, e.g. %s", n, ex.Id)
228231
}
229232

230233
select {

‎install/installer/cmd/testdata/render/agent-smith/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/installer/cmd/testdata/render/aws-setup/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/installer/cmd/testdata/render/custom-pull-repository/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/installer/cmd/testdata/render/customization/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/installer/cmd/testdata/render/external-registry/output.golden

Lines changed: 6 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)