Frontend development: redirect instead of proxy (#19177)

* [dev] make-user-admin: Grant all admin permissions

* [dev] Change frontend-dev to redirect instead of transparently proxy

This removes the "proxy needs network access to upstream" constraint, by relying on the developers browser

* [dev] front-end dev: more convenience, and added to readme

Author: geropl

components/BUILD.yaml

@@ -268,7 +268,7 @@ scripts:
         echo "No input."
       else
         echo "User: $user"
-        query="update d_b_user set rolesOrPermissions = '[\"admin\"]', fgaRelationshipsVersion=0 where name=\"$user\";"
+        query="update d_b_user set rolesOrPermissions = '[\"admin\", \"admin-permissions\"]', fgaRelationshipsVersion=0 where name=\"$user\";"
         mysql -e "$query" -u$DB_USERNAME -p$DB_PASSWORD -h 127.0.0.1 gitpod
       fi
       kill $PID || true

components/dashboard/README.md

@@ -36,9 +36,26 @@ After creating a new component, run the following to update the license header:
 
 ## How to develop in gitpod.io
 
+### Against any* Gitpod installation
+
+Gitpod installations have a feature that - if you are authorized - allow different versions of the dashboard. This allows for front-end development with live data and super-quick turnarounds.
+
+**Preconditions**
+ 1. logged in user on the respective Gitpod installation (e.g. gitpod.example.org)
+ 1. user has the `"developer"` role
+
+**Steps**
+ 1. Start a workspace (on any installation), and start the dev-server with `yarn start-local`
+ 1. Configure your browser to always send header `X-Frontend-Dev-URL` with value set to the result of `gp url 3000` to the Gitpod installation you want to modify (gitpod.example.org)
+ 1. Visit https://gitpod.example.org, start modifying your `dashboard` in your workspace, and experience the effect live (incl. hot reloading)
+
+*: This feature is _not_ enabled on all installations, and requires special user privileges.
+
+### Outdated, in-workspace (?)
+
 All the commands in this section are meant to be executed from the `components/dashboard` directory.
 
-### 1. Environment variables
+#### 1. Environment variables
 
 Set the following 2 [environment variables](https://www.gitpod.io/docs/environment-variables) either [via your account settings](https://gitpod.io/variables) or [via the command line](https://www.gitpod.io/docs/environment-variables#using-the-command-line-gp-env).
 
@@ -63,7 +80,7 @@ Replace `AUTHENTICATION_COOKIE_VALUE` with the value of your auth cookie taken f
 | -------------------------------------------------------------------------- |
 | ![Where to get the auth cookie name and value from](how-to-get-cookie.png) |
 
-### 2. Start the dashboard app
+#### 2. Start the dashboard app
 
 🚀 After following the above steps, run `yarn run start` to start developing.
 You can view the dashboard at https://`PORT_NUMBER`-`GITPOD_WORKSPACE_URL` (`PORT_NUMBER` is usually `3000`).

components/dashboard/craco.config.js

@@ -7,6 +7,10 @@ const { when } = require("@craco/craco");
 const path = require("path");
 const webpack = require("webpack");
 
+function withEndingSlash(str) {
+    return str.endsWith("/") ? str : str + "/";
+}
+
 module.exports = {
     style: {
         postcss: {
@@ -49,6 +53,16 @@ module.exports = {
                     Buffer: ["buffer", "Buffer"],
                 }),
             ],
+            // If ASSET_PATH is set, we imply that we also want a statically named main.js, so we can reference it from the outside
+            output: !!process.env.ASSET_PATH
+                ? {
+                      ...(webpack?.configure?.output || {}),
+                      filename: (pathData) => {
+                          return pathData.chunk.name === "main" ? "static/js/main.js" : undefined;
+                      },
+                      publicPath: withEndingSlash(process.env.ASSET_PATH),
+                  }
+                : undefined,
         },
     },
     devServer: {

components/dashboard/package.json

@@ -96,7 +96,7 @@
   },
   "scripts": {
     "start": "BROWSER=none HMR_HOST=`gp url 3001` craco start",
-    "start-local": "BROWSER=none HMR_HOST=`gp url 3000` craco start",
+    "start-local": "gp ports visibility 3000:public; BROWSER=none HMR_HOST=`gp url 3000` ASSET_PATH=`gp url 3000` craco start",
     "build": "craco build --verbose",
     "lint": "eslint --max-warnings=0 --ext=.jsx,.js,.tsx,.ts ./src",
     "test": "yarn test:unit",

components/proxy/conf/Caddyfile

@@ -376,7 +376,9 @@ https://{$GITPOD_DOMAIN} {
 				header_up -Upgrade
 			}
 			# Then handle it with our plugin!
-			gitpod.frontend_dev
+			gitpod.frontend_dev {
+				upstream http://dashboard.{$KUBE_NAMESPACE}.{$KUBE_DOMAIN}:3001
+			}
 		}
 
 		reverse_proxy dashboard.{$KUBE_NAMESPACE}.{$KUBE_DOMAIN}:3001 {

components/proxy/plugins/frontend_dev/frontend_dev.go

@@ -2,14 +2,17 @@
 // Licensed under the GNU Affero General Public License (AGPL).
 // See License.AGPL.txt in the project root for license information.
 
-package workspacedownload
+package frontend_dev
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"regexp"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
@@ -25,24 +28,26 @@ const (
 )
 
 func init() {
-	caddy.RegisterModule(Config{})
+	caddy.RegisterModule(FrontendDev{})
 	httpcaddyfile.RegisterHandlerDirective(frontendDevModule, parseCaddyfile)
 }
 
-// Config implements an HTTP handler that extracts gitpod headers
-type Config struct {
+// FrontendDev implements an HTTP handler that extracts gitpod headers
+type FrontendDev struct {
+	Upstream    string `json:"upstream,omitempty"`
+	UpstreamUrl *url.URL
 }
 
 // CaddyModule returns the Caddy module information.
-func (Config) CaddyModule() caddy.ModuleInfo {
+func (FrontendDev) CaddyModule() caddy.ModuleInfo {
 	return caddy.ModuleInfo{
 		ID:  "http.handlers.frontend_dev",
-		New: func() caddy.Module { return new(Config) },
+		New: func() caddy.Module { return new(FrontendDev) },
 	}
 }
 
 // ServeHTTP implements caddyhttp.MiddlewareHandler.
-func (m Config) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
+func (m FrontendDev) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
 	enabled := os.Getenv(frontendDevEnabledEnvVarName)
 	if enabled != "true" {
 		caddy.Log().Sugar().Debugf("Dev URL header present but disabled")
@@ -60,70 +65,127 @@ func (m Config) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp
 		return caddyhttp.Error(http.StatusInternalServerError, fmt.Errorf("unexpected error forwarding to dev URL"))
 	}
 
-	targetQuery := devURL.RawQuery
 	director := func(req *http.Request) {
-		req.URL.Scheme = devURL.Scheme
-		req.URL.Host = devURL.Host
-		req.Host = devURL.Host // override host header so target proxy can handle this request properly
-
-		req.URL.Path, req.URL.RawPath = joinURLPath(devURL, req.URL)
-		if targetQuery == "" || req.URL.RawQuery == "" {
-			req.URL.RawQuery = targetQuery + req.URL.RawQuery
-		} else {
-			req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
-		}
+		req.URL.Scheme = m.UpstreamUrl.Scheme
+		req.URL.Host = m.UpstreamUrl.Host
+		req.Host = m.UpstreamUrl.Host
 		if _, ok := req.Header["User-Agent"]; !ok {
 			// explicitly disable User-Agent so it's not set to default value
 			req.Header.Set("User-Agent", "")
 		}
+		req.Header.Set("Accept-Encoding", "") // we can't handle other than plain text
+		// caddy.Log().Sugar().Infof("director request (mod): %v", req.URL.String())
 	}
-	proxy := httputil.ReverseProxy{Director: director}
+	proxy := httputil.ReverseProxy{Director: director, Transport: &RedirectingTransport{baseUrl: devURL}}
 	proxy.ServeHTTP(w, r)
 
 	return nil
 }
 
-func joinURLPath(a, b *url.URL) (path, rawpath string) {
-	if a.RawPath == "" && b.RawPath == "" {
-		return singleJoiningSlash(a.Path, b.Path), ""
+type RedirectingTransport struct {
+	baseUrl *url.URL
+}
+
+func (rt *RedirectingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	// caddy.Log().Sugar().Infof("issuing upstream request: %s", req.URL.Path)
+	resp, err := http.DefaultTransport.RoundTrip(req)
+	if err != nil {
+		return nil, err
 	}
-	// Same as singleJoiningSlash, but uses EscapedPath to determine
-	// whether a slash should be added
-	apath := a.EscapedPath()
-	bpath := b.EscapedPath()
-
-	aslash := strings.HasSuffix(apath, "/")
-	bslash := strings.HasPrefix(bpath, "/")
-
-	switch {
-	case aslash && bslash:
-		return a.Path + b.Path[1:], apath + bpath[1:]
-	case !aslash && !bslash:
-		return a.Path + "/" + b.Path, apath + "/" + bpath
+
+	// gpl: Do we have better means to avoid checking the body?
+	if resp.StatusCode < 300 && strings.HasPrefix(resp.Header.Get("Content-type"), "text/html") {
+		// caddy.Log().Sugar().Infof("trying to match request: %s", req.URL.Path)
+		modifiedResp := MatchAndRewriteRootRequest(resp, rt.baseUrl)
+		if modifiedResp != nil {
+			caddy.Log().Sugar().Debugf("using modified upstream response: %s", req.URL.Path)
+			return modifiedResp, nil
+		}
 	}
-	return a.Path + b.Path, apath + bpath
+	caddy.Log().Sugar().Debugf("forwarding upstream response: %s", req.URL.Path)
+
+	return resp, nil
 }
 
-func singleJoiningSlash(a, b string) string {
-	aslash := strings.HasSuffix(a, "/")
-	bslash := strings.HasPrefix(b, "/")
-	switch {
-	case aslash && bslash:
-		return a + b[1:]
-	case !aslash && !bslash:
-		return a + "/" + b
+func MatchAndRewriteRootRequest(or *http.Response, baseUrl *url.URL) *http.Response {
+	// match index.html?
+	prefix := []byte("<!doctype html>")
+	var buf bytes.Buffer
+	bodyReader := io.TeeReader(or.Body, &buf)
+	prefixBuf := make([]byte, len(prefix))
+	_, err := io.ReadAtLeast(bodyReader, prefixBuf, len(prefix))
+	if err != nil {
+		caddy.Log().Sugar().Debugf("prefix match: can't read response body: %w", err)
+		return nil
+	}
+	if !bytes.Equal(prefix, prefixBuf) {
+		caddy.Log().Sugar().Debugf("prefix mismatch: %s", string(prefixBuf))
+		return nil
 	}
-	return a + b
+
+	caddy.Log().Sugar().Debugf("match index.html")
+	_, err = io.Copy(&buf, or.Body)
+	if err != nil {
+		caddy.Log().Sugar().Debugf("unable to copy response body: %w, path: %s", err, or.Request.URL.Path)
+		return nil
+	}
+	fullBody := buf.String()
+
+	mainJs := regexp.MustCompile(`"[^"]+?main\.[0-9a-z]+\.js"`)
+	fullBody = mainJs.ReplaceAllStringFunc(fullBody, func(s string) string {
+		return fmt.Sprintf(`"%s/static/js/main.js"`, baseUrl.String())
+	})
+
+	mainCss := regexp.MustCompile(`<link[^>]+?rel="stylesheet">`)
+	fullBody = mainCss.ReplaceAllString(fullBody, "")
+
+	hrefs := regexp.MustCompile(`href="/`)
+	fullBody = hrefs.ReplaceAllString(fullBody, fmt.Sprintf(`href="%s/`, baseUrl.String()))
+
+	or.Body = io.NopCloser(strings.NewReader(fullBody))
+	or.Header.Set("Content-Length", fmt.Sprintf("%d", len(fullBody)))
+	or.Header.Set("Etag", "")
+	return or
 }
 
 // UnmarshalCaddyfile implements Caddyfile.Unmarshaler.
-func (m *Config) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+func (m *FrontendDev) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	if !d.Next() {
+		return d.Err("expected token following filter")
+	}
+
+	for d.NextBlock(0) {
+		key := d.Val()
+		var value string
+		d.Args(&value)
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+
+		switch key {
+		case "upstream":
+			m.Upstream = value
+
+		default:
+			return d.Errf("unrecognized subdirective '%s'", value)
+		}
+	}
+
+	if m.Upstream == "" {
+		return fmt.Errorf("frontend_dev: 'upstream' config field may not be empty")
+	}
+
+	upstreamURL, err := url.Parse(m.Upstream)
+	if err != nil {
+		return fmt.Errorf("frontend_dev: 'upstream' is not a valid URL: %w", err)
+	}
+	m.UpstreamUrl = upstreamURL
 
 	return nil
 }
 
 func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
-	m := new(Config)
+	m := new(FrontendDev)
 	err := m.UnmarshalCaddyfile(h.Dispenser)
 	if err != nil {
 		return nil, err
@@ -134,6 +196,6 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 
 // Interface guards
 var (
-	_ caddyhttp.MiddlewareHandler = (*Config)(nil)
-	_ caddyfile.Unmarshaler       = (*Config)(nil)
+	_ caddyhttp.MiddlewareHandler = (*FrontendDev)(nil)
+	_ caddyfile.Unmarshaler       = (*FrontendDev)(nil)
 )