Merge branch 'gitpod-io:main' into hide-editor-warning

Author: Devansu-Yadav

.github/CODEOWNERS

@@ -11,7 +11,6 @@
 /components/dashboard @gitpod-io/engineering-webapp @gtsiolis
 /components/docker-up @gitpod-io/engineering-workspace
 /components/ee/agent-smith @gitpod-io/engineering-workspace
-/components/ee/payment-endpoint @gitpod-io/engineering-webapp
 /components/gitpod-cli @gitpod-io/engineering-ide
 /components/gitpod-db @gitpod-io/engineering-webapp
 /components/gitpod-messagebus @gitpod-io/engineering-webapp
@@ -129,3 +128,18 @@
 /CHANGELOG.md
 /components/ide/jetbrains/backend-plugin/gradle-latest.properties
 /components/ide/jetbrains/gateway-plugin/gradle-latest.properties
+
+#
+# Add so that teams assert we're not breaking each other's integration tests
+/test/pkg/agent @gitpod-io/engineering-workspace
+/test/pkg/integration @gitpod-io/engineering-ide @gitpod-io/engineering-workspace
+/test/pkg/report @gitpod-io/engineering-workspace
+/test/tests/workspace @gitpod-io/engineering-workspace
+/test/tests/smoke-test @gitpod-io/engineering-ide @gitpod-io/engineering-workspace
+/test/tests/ide @gitpod-io/engineering-ide
+/test/tests/components/content-service @gitpod-io/engineering-workspace
+/test/tests/components/database @gitpod-io/engineering-webapp
+/test/tests/components/image-builder @gitpod-io/engineering-workspace
+/test/tests/components/server @gitpod-io/engineering-webapp
+/test/tests/components/ws-daemon @gitpod-io/engineering-workspace
+/test/tests/components/ws-manager @gitpod-io/engineering-workspace

.github/actions/deploy-gitpod/metadata.yml

@@ -17,7 +17,7 @@ inputs:
         description: "Use WS Manager MK2"
         required: false
     with_dedicated_emu:
-        description: "Dedicated Auth"
+        description: "Dedicated Config"
         required: false
     with_ee_licencse:
         description: "Use EE license"

.github/pull_request_template.md

@@ -42,7 +42,7 @@ gitpod:summary
 
 - [ ] analytics=segment
 - [ ] with-dedicated-emulation
-- [ ] with-ws-manager-mk2
+- [x] with-ws-manager-mk2
 - [ ] workspace-feature-flags
   Add desired feature flags to the end of the line above, space separated
 </details>

.github/workflows/build.yml

@@ -23,14 +23,14 @@ jobs:
       is_main_branch: ${{ (github.head_ref || github.ref) == 'refs/heads/main' }}
       version: ${{ steps.branches.outputs.sanitized-branch-name }}-gha.${{github.run_number}}
       preview_enable: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft with-preview') }}
-      preview_infra_provider: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft with-gce-vm') && 'gce' || 'harvester' }}
+      preview_infra_provider: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft with-gce-vm') && 'gce' || 'harvester' }}
       build_no_cache: ${{ contains( steps.pr-details.outputs.pr_body, '[x] leeway-no-cache') }}
       build_no_test: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft no-test') }}
-      with_large_vm: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft with-large-vm') }}
-      publish_to_npm: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft publish-to-npm') }}
-      publish_to_jbmp: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft publish-to-jb-marketplace') }}
-      with_ws_manager_mk2: ${{ contains( steps.pr-details.outputs.pr_body, '[X] with-ws-manager-mk2') }}
-      with_dedicated_emulation: ${{ contains( steps.pr-details.outputs.pr_body, '[X] with-dedicated-emulation') }}
+      with_large_vm: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft with-large-vm') }}
+      publish_to_npm: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft publish-to-npm') }}
+      publish_to_jbmp: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft publish-to-jb-marketplace') }}
+      with_ws_manager_mk2: ${{ contains( steps.pr-details.outputs.pr_body, '[x] with-ws-manager-mk2') }}
+      with_dedicated_emulation: ${{ contains( steps.pr-details.outputs.pr_body, '[x] with-dedicated-emulation') }}
       analytics: ${{ steps.output.outputs.analytics }}
       workspace_feature_flags: ${{ steps.output.outputs.workspace_feature_flags }}
       pr_no_diff_skip: ${{ steps.pr-diff.outputs.pr_no_diff_skip }}
@@ -158,7 +158,7 @@ jobs:
     container:
       image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:kylos101-prebuild-jam-gha.5832
       volumes:
-        - /var/tmp:/var/tmp
+        - /var/tmp/${{ needs.configuration.outputs.leeway_cache_bucket }}:/var/tmp
         - /tmp:/tmp
       env:
         DB_HOST: "mysql"
@@ -343,7 +343,7 @@ jobs:
           sa_key: ${{ secrets.GCP_CREDENTIALS }}
           previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }}
           wsmanager_mk2: ${{needs.configuration.outputs.with_ws_manager_mk2}}
-          with_dedicated_emulation: ${{needs.configuration.outputs.with_dedicated_emulation}}
+          with_dedicated_emu: ${{needs.configuration.outputs.with_dedicated_emulation}}
           analytics: ${{needs.configuration.outputs.analytics}}
           workspace_feature_flags: ${{needs.configuration.outputs.workspace_feature_flags}}
       - uses: actions/github-script@v6
@@ -414,6 +414,7 @@ jobs:
             TEST_BUILD_ID: ${{ github.run_id }}
             TEST_BUILD_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
             TEST_BUILD_REF: ${{ github.head_ref || github.ref }}
+            WS_MANAGER_MK2: ${{ needs.configuration.outputs.with_ws_manager_mk2 != 'false' }}
         run: |
             set -euo pipefail
 

.github/workflows/code-nightly.yml

@@ -31,7 +31,7 @@ jobs:
       - run: |
           gcloud auth configure-docker --quiet
           export LEEWAY_WORKSPACE_ROOT=$(pwd)
-          codeHeadCommit=$(curl -H 'Accept: application/vnd.github.VERSION.sha' https://api.github.com/repos/gitpod-io/openvscode-server/commits/gp-code/release/1.77)
+          codeHeadCommit=$(curl -H 'Accept: application/vnd.github.VERSION.sha' https://api.github.com/repos/gitpod-io/openvscode-server/commits/gp-code/main)
           codeVersion=$(curl https://raw.githubusercontent.com/gitpod-io/openvscode-server/$codeHeadCommit/package.json | jq .version)
           cd components/ide/code
           leeway build -Dversion=nightly -DimageRepoBase=eu.gcr.io/gitpod-core-dev/build -DcodeCommit=$codeHeadCommit -DcodeVersion=$codeVersion -DcodeQuality=insider .:docker

.github/workflows/ide-integration-tests.yml

@@ -8,6 +8,11 @@ on:
             version:
                 required: true
                 description: "The version of Gitpod to install"
+            wsman_mk2:
+                required: false
+                type: boolean
+                default: true
+                description: "Run tests against ws-manager-mk2"
             skip_deploy:
                 required: false
                 description: "Skip deploy preview environment (debug only)"
@@ -73,7 +78,7 @@ jobs:
               with:
                   name: ${{ needs.configuration.outputs.name }}
                   sa_key: ${{ secrets.GCP_CREDENTIALS }}
-                  infrastructure_provider: harvester
+                  infrastructure_provider: gce
                   large_vm: true
             - name: Deploy Gitpod to the preview environment
               id: deploy-gitpod
@@ -83,6 +88,7 @@ jobs:
                   name: ${{ needs.configuration.outputs.name }}
                   sa_key: ${{ secrets.GCP_CREDENTIALS }}
                   version: ${{ needs.configuration.outputs.version}}
+                  wsmanager_mk2: ${{ github.event.inputs.wsman_mk2 != 'false' }}
 
     check:
         name: Check for regressions
@@ -103,6 +109,9 @@ jobs:
                   USER_TOKEN: ${{ secrets.IDE_INTEGRATION_TEST_USER_TOKEN }}
                   PREVIEW_ENV_DEV_SA_KEY: ${{ secrets.GCP_CREDENTIALS }}
                   PREVIEW_NAME: ${{ needs.configuration.outputs.name }}
+                  TEST_BUILD_ID: ${{ github.run_id }}
+                  TEST_BUILD_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+                  TEST_BUILD_REF: ${{ github.head_ref || github.ref }}
               run: |
                   set -euo pipefail
 
@@ -132,6 +141,8 @@ jobs:
                   VSCODE_TESTS="$IDE_TESTS_DIR/vscode"
                   SSH_TESTS="$IDE_TESTS_DIR/ssh"
 
+                  export WS_MANAGER_MK2="${{ github.event.inputs.wsman_mk2 != 'false' }}"
+
                   go install github.com/jstemmer/go-junit-report/v2@latest
 
                   FAILURE_COUNT=0

.github/workflows/jetbrains-update-plugin-platform-template.yml

@@ -106,6 +106,7 @@ jobs:
                       - [x] /werft with-gce-vm
                       - [x] with-integration-tests=jetbrains
                       - [x] latest-ide-version=${{ contains(inputs.pluginId, 'latest') }}
+                      - [x] with-ws-manager-mk2
 
                       _This PR was created automatically with GitHub Actions using [this](https://github.com/gitpod-io/gitpod/blob/main/.github/workflows/jetbrains-update-plugin-platform-template.yml) template._
                   commit-message: "Update Platform Version of ${{ inputs.pluginName }} to ${{ steps.latest-version.outputs.result }}"

.github/workflows/jetbrains-updates.yml

@@ -67,6 +67,7 @@ jobs:
                       - [x] /werft with-gce-vm
                       - [x] with-integration-tests=jetbrains
                       - [x] latest-ide-version=false
+                      - [x] with-ws-manager-mk2
 
                       _This PR was created automatically with GitHub Actions using [this](https://github.com/gitpod-io/gitpod/blob/main/.github/workflows/jetbrains-updates.yml) GHA_
                   commit-message: "[JetBrains] Update IDE images to new build version"

.github/workflows/preview-env-check-regressions.yml

@@ -15,7 +15,12 @@ on:
             infrastructure_provider:
                 description: "The infrastructure provider to use. Valid options: harvester, gcp"
                 required: false
-                default: harvester
+                default: gcp
+            wsman_mk2:
+                required: false
+                type: boolean
+                default: true
+                description: "Run tests against ws-manager-mk2"
 
 jobs:
     configuration:
@@ -71,6 +76,7 @@ jobs:
                   name: ${{ needs.configuration.outputs.name }}
                   sa_key: ${{ secrets.GCP_CREDENTIALS }}
                   version: ${{ needs.configuration.outputs.version}}
+                  wsmanager_mk2: ${{ github.event.inputs.wsman_mk2 != 'false' }}
 
     check:
         name: Check for regressions
@@ -118,6 +124,8 @@ jobs:
 
                   TESTS_DIR="$GITHUB_WORKSPACE/test/tests/smoke-test"
 
+                  export WS_MANAGER_MK2="${{ github.event.inputs.wsman_mk2 != 'false' }}"
+
                   go install github.com/jstemmer/go-junit-report/v2@latest
 
                   echo "running integration for smoke test"

.github/workflows/update-image-digest.yml

@@ -0,0 +1,85 @@
+name: Update container images digest
+
+on:
+  workflow_dispatch:
+  schedule:
+    # At the start of every day
+    - cron: "0 0 * * *"
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set git identity
+        run: |
+          git config --global user.name $GITHUB_USER
+          git config --global user.email $GITHUB_EMAIL
+        env:
+          GITHUB_USER: roboquat
+          GITHUB_EMAIL: [email protected]
+
+      - uses: imjasonh/[email protected]
+
+      - name: Check if an update is available
+        shell: bash
+        run: |
+          while IFS= read -r -d '' file; do
+            if [[ "$file" == *testdata* ]]; then
+              echo "Skipping testdata ${file}"
+              continue
+            fi
+            images=$(grep -i -E '[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*@sha256:[a-z0-9]+' "$file" | cut -d @ -f1 | rev | cut -d ' ' -f1 | cut -d '"' -f1 | rev | sed -e "s/^docker:\/\///" | tr '\n' ',' || true)
+            digests=$(grep -i -E '[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*@sha256:[a-z0-9]+' "$file" | cut -d @ -f2 | cut -d ' ' -f1 | cut -d '"' -f1 | tr '\n' ',' || true)
+            IFS=',' read -r -a images2 <<< "$images"
+            IFS=',' read -r -a digests2 <<< "$digests"
+
+            if [ -n "$images" ]; then
+                for i in "${!images2[@]}"; do
+                  if [[ ${images2[i]} != *":"* ]]; then
+                    echo "Image ${images2[i]} in file $file does not have a tag, ignoring..."
+                    continue
+                  fi
+                  if [[ ${images2[i]} == *\.local:* ]]; then
+                    echo "Skipping local registry image ${images2[i]}"
+                    continue
+                  fi
+                  echo "Processing ${images2[i]} in file $file"
+                  updated_digest=$(crane digest "${images2[i]}")
+                  if [ "$updated_digest" != "${digests2[i]}" ] && [ -n "$updated_digest" ]; then
+                    echo "Digest ${digests2[i]} for image ${images2[i]} is different, new digest is $updated_digest, updating..."
+                    sed -i -e "s/${digests2[i]}/$updated_digest/g" "$file"
+                  fi
+                done
+            fi
+          done < <(find "$(pwd)" -type f \( -name "*.yaml" -o -name "*.yml" -o -name "Dockerfile*" -o -name "leeway.Dockerfile" \) -print0)
+
+      - name: Check workspace
+        id: create_pr
+        shell: bash
+        run: |
+          if [[ $(git diff --stat) != '' ]]; then
+            echo "create_pr=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@923ad837f191474af6b1721408744feb989a4c27 # v4
+        if: ${{ steps.create_pr.outputs.create_pr == 'true' }}
+        with:
+          token: ${{ secrets.ROBOQUAT_AUTOMATIC_CHANGELOG }}
+          commit-message: update index
+          title: "Update images digests"
+          body: |
+            Update images digests using the latest version available for image/s
+
+            ```release-note
+            NONE
+            ```
+          labels: automated pr, kind/cleanup, release-note-none
+          branch: update-digests
+          delete-branch: true