[server] maintain workspace instance state

get closer to workspace instance state reflects actual state in cluster.

Author: svenefftinge

components/server/src/workspace/workspace-starter.ts

@@ -522,12 +522,15 @@ export class WorkspaceStarter {
                 }
             } catch (err) {
                 increaseFailedInstanceStartCounter("startOnClusterFailed");
+                await this.failInstanceStart({ span }, err, workspace, instance);
                 throw err;
             }
 
             if (!resp) {
                 increaseFailedInstanceStartCounter("clusterSelectionFailed");
-                throw new Error("cannot start a workspace because no workspace clusters are available");
+                const err = new Error("cannot start a workspace because no workspace clusters are available");
+                await this.failInstanceStart({ span }, err, workspace, instance);
+                throw err;
             }
             increaseSuccessfulInstanceStartCounter(retries);
 
@@ -570,9 +573,6 @@ export class WorkspaceStarter {
 
             return { instanceID: instance.id, workspaceURL: resp.url };
         } catch (err) {
-            TraceContext.setError({ span }, err);
-            await this.failInstanceStart({ span }, err, workspace, instance);
-
             if (rethrow) {
                 throw err;
             } else {

components/ws-manager-bridge/src/bridge.ts

@@ -346,6 +346,15 @@ export class WorkspaceManagerBridge implements Disposable {
                     }
 
                     instance.status.phase = "running";
+                    // let's check if the state is inconsistent and be loud if it is.
+                    if (instance.stoppedTime || instance.stoppingTime) {
+                        log.error("Resetting already stopped workspace to running.", {
+                            instanceId: instance.id,
+                            stoppedTime: instance.stoppedTime,
+                        });
+                        instance.stoppedTime = undefined;
+                        instance.stoppingTime = undefined;
+                    }
                     break;
                 case WorkspacePhase.INTERRUPTED:
                     instance.status.phase = "interrupted";