Skip to content

[installer] Do not pull blobserve implementation into installer #16729

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 8, 2023
Merged

[installer] Do not pull blobserve implementation into installer #16729

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6f17db4
[installer] Do not pull blobserve implementation into installer
easyCZ Mar 8, 2023
52cebf3
fix
easyCZ Mar 8, 2023
cb3b460
fix
easyCZ Mar 8, 2023
4e13237
Fix
easyCZ Mar 8, 2023
92a69e8
fix
easyCZ Mar 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions components/blobserve/cmd/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
"github.com/containerd/containerd/remotes/docker"
"github.com/docker/cli/cli/config/configfile"
"github.com/docker/distribution/reference"
"github.com/gitpod-io/gitpod/blobserve/pkg/config"
blobserve_config "github.com/gitpod-io/gitpod/blobserve/pkg/config"
"github.com/heptiolabs/healthcheck"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
Expand All @@ -42,7 +42,7 @@ var runCmd = &cobra.Command{
Short: "Starts the blobserve",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
cfg, err := config.GetConfig(args[0])
cfg, err := blobserve_config.GetConfig(args[0])
if err != nil {
log.WithError(err).WithField("filename", args[0]).Fatal("cannot load config")
}
Expand Down
44 changes: 5 additions & 39 deletions components/blobserve/pkg/blobserve/blobserve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,16 @@ import (
"github.com/gorilla/mux"
"golang.org/x/xerrors"

blobserve_config "github.com/gitpod-io/gitpod/blobserve/pkg/config"
"github.com/gitpod-io/gitpod/common-go/log"
"github.com/gitpod-io/gitpod/common-go/util"
)

// ResolverProvider provides new resolver
type ResolverProvider func() remotes.Resolver

// Server offers image blobs for download
type Server struct {
Config Config
Config blobserve_config.BlobServe
Resolver ResolverProvider

refstore *refstore
Expand All @@ -44,40 +44,6 @@ type BlobserveInlineVars struct {
SupervisorImage string `json:"supervisor"`
}

type BlobSpace struct {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is moved to the config package

Location string `json:"location"`
MaxSize int64 `json:"maxSizeBytes,omitempty"`
}

type Repo struct {
PrePull []string `json:"prePull,omitempty"`
Workdir string `json:"workdir,omitempty"`
Replacements []StringReplacement `json:"replacements,omitempty"`
InlineStatic []InlineReplacement `json:"inlineStatic,omitempty"`
}

// Config configures a server.
type Config struct {
Port int `json:"port"`
Timeout util.Duration `json:"timeout,omitempty"`
Repos map[string]Repo `json:"repos"`
// AllowAnyRepo enables users to access any repo/image, irregardles if they're listed in the
// ref config or not.
AllowAnyRepo bool `json:"allowAnyRepo"`
BlobSpace BlobSpace `json:"blobSpace"`
}

type StringReplacement struct {
Path string `json:"path"`
Search string `json:"search"`
Replacement string `json:"replacement"`
}

type InlineReplacement struct {
Search string `json:"search"`
Replacement string `json:"replacement"`
}

// From https://github.com/distribution/distribution/blob/v2.7.1/reference/regexp.go
var match = regexp.MustCompile

Expand Down Expand Up @@ -115,7 +81,7 @@ var ReferenceRegexp = expression(reference.NameRegexp,
optional(literal("@"), reference.DigestRegexp))

// NewServer creates a new blob server
func NewServer(cfg Config, resolver ResolverProvider) (*Server, error) {
func NewServer(cfg blobserve_config.BlobServe, resolver ResolverProvider) (*Server, error) {
refstore, err := newRefStore(cfg, resolver)
if err != nil {
return nil, err
Expand Down Expand Up @@ -202,7 +168,7 @@ func (reg *Server) serve(w http.ResponseWriter, req *http.Request) {
repo := pref.Name()

var workdir string
var inlineReplacements []InlineReplacement
var inlineReplacements []blobserve_config.InlineReplacement
if cfg, ok := reg.Config.Repos[repo]; ok {
workdir = cfg.Workdir
inlineReplacements = cfg.InlineStatic
Expand Down Expand Up @@ -285,7 +251,7 @@ func (reg *Server) serve(w http.ResponseWriter, req *http.Request) {
http.StripPrefix(pathPrefix, http.FileServer(fs)).ServeHTTP(w, req)
}

func inlineVars(req *http.Request, r io.ReadSeeker, inlineReplacements []InlineReplacement) (io.ReadSeeker, error) {
func inlineVars(req *http.Request, r io.ReadSeeker, inlineReplacements []blobserve_config.InlineReplacement) (io.ReadSeeker, error) {
inlineVarsValue := req.Header.Get("X-BlobServe-InlineVars")
if len(inlineReplacements) == 0 || inlineVarsValue == "" {
return r, nil
Expand Down
7 changes: 4 additions & 3 deletions components/blobserve/pkg/blobserve/blobspace_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"path/filepath"
"testing"

blobserve_config "github.com/gitpod-io/gitpod/blobserve/pkg/config"
"github.com/google/go-cmp/cmp"
)

Expand Down Expand Up @@ -126,7 +127,7 @@ func Test_inlineVars(t *testing.T) {
tests := []struct {
Name string
InlineVars *BlobserveInlineVars
Replacements []InlineReplacement
Replacements []blobserve_config.InlineReplacement
Content string
Expected string
}{
Expand All @@ -141,7 +142,7 @@ func Test_inlineVars(t *testing.T) {
},
{
Name: "no inline vars",
Replacements: []InlineReplacement{
Replacements: []blobserve_config.InlineReplacement{
{Search: "aaa", Replacement: "${ide}"},
{Search: "bbb", Replacement: "${supervisor}"},
},
Expand All @@ -154,7 +155,7 @@ func Test_inlineVars(t *testing.T) {
IDE: "foo",
SupervisorImage: "bar",
},
Replacements: []InlineReplacement{
Replacements: []blobserve_config.InlineReplacement{
{Search: "aaa", Replacement: "${ide}"},
{Search: "bbb", Replacement: "${supervisor}"},
},
Expand Down
3 changes: 2 additions & 1 deletion components/blobserve/pkg/blobserve/refstore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
ociv1 "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/xerrors"

blobserve_config "github.com/gitpod-io/gitpod/blobserve/pkg/config"
"github.com/gitpod-io/gitpod/common-go/log"
"github.com/gitpod-io/gitpod/registry-facade/pkg/registry"
)
Expand Down Expand Up @@ -53,7 +54,7 @@ type refstore struct {
once *sync.Once
}

func newRefStore(cfg Config, resolver ResolverProvider) (*refstore, error) {
func newRefStore(cfg blobserve_config.BlobServe, resolver ResolverProvider) (*refstore, error) {
bs, err := newBlobSpace(cfg.BlobSpace.Location, cfg.BlobSpace.MaxSize, 10*time.Minute)
if err != nil {
return nil, err
Expand Down
41 changes: 41 additions & 0 deletions components/blobserve/pkg/config/blobserve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Copyright (c) 2023 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License.AGPL.txt in the project root for license information.

package blobserve_config

import "github.com/gitpod-io/gitpod/common-go/util"

// Config configures a server.
type BlobServe struct {
Port int `json:"port"`
Timeout util.Duration `json:"timeout,omitempty"`
Repos map[string]Repo `json:"repos"`
// AllowAnyRepo enables users to access any repo/image, irregardles if they're listed in the
// ref config or not.
AllowAnyRepo bool `json:"allowAnyRepo"`
BlobSpace BlobSpace `json:"blobSpace"`
}

type StringReplacement struct {
Path string `json:"path"`
Search string `json:"search"`
Replacement string `json:"replacement"`
}

type InlineReplacement struct {
Search string `json:"search"`
Replacement string `json:"replacement"`
}

type Repo struct {
PrePull []string `json:"prePull,omitempty"`
Workdir string `json:"workdir,omitempty"`
Replacements []StringReplacement `json:"replacements,omitempty"`
InlineStatic []InlineReplacement `json:"inlineStatic,omitempty"`
}

type BlobSpace struct {
Location string `json:"location"`
MaxSize int64 `json:"maxSizeBytes,omitempty"`
}
14 changes: 6 additions & 8 deletions components/blobserve/pkg/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,20 @@
// Licensed under the GNU Affero General Public License (AGPL).
// See License.AGPL.txt in the project root for license information.

package config
package blobserve_config

import (
"encoding/json"
"os"

"github.com/gitpod-io/gitpod/blobserve/pkg/blobserve"
)

// Config configures this service
type Config struct {
BlobServe blobserve.Config `json:"blobserve"`
AuthCfg string `json:"dockerAuth"`
PProfAddr string `json:"pprofAddr"`
PrometheusAddr string `json:"prometheusAddr"`
ReadinessProbeAddr string `json:"readinessProbeAddr"`
BlobServe BlobServe `json:"blobserve"`
AuthCfg string `json:"dockerAuth"`
PProfAddr string `json:"pprofAddr"`
PrometheusAddr string `json:"prometheusAddr"`
ReadinessProbeAddr string `json:"readinessProbeAddr"`
}

// getConfig loads and validates the configuration
Expand Down
63 changes: 4 additions & 59 deletions install/installer/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,10 @@ require (
sigs.k8s.io/yaml v1.3.0
)

require cloud.google.com/go/compute/metadata v0.2.1 // indirect
require (
cloud.google.com/go/compute/metadata v0.2.1 // indirect
github.com/frankban/quicktest v1.14.3 // indirect
)

require (
cloud.google.com/go v0.105.0 // indirect
Expand Down Expand Up @@ -106,11 +109,8 @@ require (
github.com/containerd/typeurl v1.0.2 // indirect
github.com/containers/storage v1.39.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/docker/cli v20.10.21+incompatible // indirect
github.com/docker/docker v20.10.17+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
Expand All @@ -131,13 +131,11 @@ require (
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fvbommel/sortorder v1.0.1 // indirect
github.com/gitpod-io/gitpod/content-service v0.0.0-00010101000000-000000000000 // indirect
github.com/gitpod-io/gitpod/registry-facade v0.0.0-00010101000000-000000000000 // indirect
github.com/gitpod-io/gitpod/usage-api v0.0.0-00010101000000-000000000000 // indirect
github.com/gitpod-io/golang-crypto v0.0.0-20220823040820-b59f56dfbab3 // indirect
github.com/go-errors/errors v1.0.1 // indirect
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
Expand All @@ -146,7 +144,6 @@ require (
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-redis/redis/v7 v7.4.1 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/goccy/go-yaml v1.9.5 // indirect
Expand All @@ -170,40 +167,11 @@ require (
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-block-format v0.0.3 // indirect
github.com/ipfs/go-blockservice v0.5.0 // indirect
github.com/ipfs/go-cid v0.3.2 // indirect
github.com/ipfs/go-datastore v0.6.0 // indirect
github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
github.com/ipfs/go-ipfs-cmds v0.8.2 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
github.com/ipfs/go-ipfs-files v0.2.0 // indirect
github.com/ipfs/go-ipfs-http-client v0.4.0 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
github.com/ipfs/go-ipld-format v0.4.0 // indirect
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
github.com/ipfs/go-libipfs v0.2.0 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-merkledag v0.9.0 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-path v0.3.0 // indirect
github.com/ipfs/go-unixfs v0.4.2 // indirect
github.com/ipfs/go-verifcid v0.0.2 // indirect
github.com/ipfs/interface-go-ipfs-core v0.7.0 // indirect
github.com/ipld/go-codec-dagpb v1.5.0 // indirect
github.com/ipld/go-ipld-prime v0.19.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/jinzhu/copier v0.3.5 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
Expand All @@ -220,16 +188,11 @@ require (
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lib/pq v1.10.7 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-libp2p v0.23.4 // indirect
github.com/libp2p/go-libp2p-core v0.20.1 // indirect
github.com/libp2p/go-openssl v0.1.0 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-pointer v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mdlayher/netlink v1.4.2 // indirect
Expand All @@ -250,14 +213,6 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multiaddr v0.8.0 // indirect
github.com/multiformats/go-multibase v0.1.1 // indirect
github.com/multiformats/go-multicodec v0.7.0 // indirect
github.com/multiformats/go-multihash v0.2.1 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
Expand All @@ -268,23 +223,19 @@ require (
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/relvacode/iso8601 v1.1.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/robfig/cron v1.2.0 // indirect
github.com/rs/cors v1.7.0 // indirect
github.com/rs/xid v1.2.1 // indirect
github.com/rubenv/sql-migrate v1.1.1 // indirect
github.com/russross/blackfriday v1.5.2 // indirect
github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/slok/go-http-metrics v0.10.0 // indirect
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stripe/stripe-go/v72 v72.114.0 // indirect
Expand All @@ -297,19 +248,14 @@ require (
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.7.0 // indirect
go.opentelemetry.io/otel/trace v1.7.0 // indirect
go.starlark.net v0.0.0-20200821142938-949cc6f4b097 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
Expand Down Expand Up @@ -338,7 +284,6 @@ require (
k8s.io/component-base v0.25.0 // indirect
k8s.io/klog/v2 v2.80.1 // indirect
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
oras.land/oras-go v1.2.0 // indirect
sigs.k8s.io/controller-runtime v0.11.2 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
Expand Down
Loading