Skip to content

Commit 24ef38a

Browse files
gcurtisgcurtis
committed
devconfig,shellgen: option to patch ELF binaries with newer glibc
Add an optional `patch_glibc` boolean field to packages in devbox.json. When true, devbox will patch any ELF binaries in a package to link against the latest `nixpkgs#glibc` at runtime. This is a workaround for dynamic linking issues that can arise when using older packages. For example, an old Python interpreter that's linked against glibc 2.35 might run a script that loads a native library that requires glibc 2.37. Because the linker only loads a library into a process once (regardless of version), linking will fail when the script references a 2.37 symbol. The following devbox.json will reproduce a glibc version bug when the `virtenv` and `crash` scripts are run on x86-64-linux (aarch64 will not work). Setting the `patch_glibc` field to true will fix it: ```json { "packages": { "binutils": "latest", "libpqxx": "latest", "libxcrypt": "latest", "libz": "latest", "python37Packages.pip": "latest", "python": { "version": "3.7", "patch_glibc": false } }, "shell": { "scripts": { "crash": "python3 -c 'import psycopg2'", "virtenv": [ "rm -rf \"$VENV_DIR\"", "echo \"python3 path: $(readlink -f $(command -v python3))\"", "python3 -m venv \"$VENV_DIR\"", ". \"$VENV_DIR/bin/activate\"", "echo \"pip3 path: $(readlink -f $(command -v pip3))\"", "pip3 install psycopg2==2.9.5" ] } } } ``` This field is intended to be a "last resort" for when a package cannot be updated to a newer version. Upgrading to a newer version of Python in the example above is preferable.
1 parent 9444bae commit 24ef38a

File tree

7 files changed

+266
-66
lines changed

7 files changed

+266
-66
lines changed

‎internal/devconfig/packages.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,10 @@ type Package struct {
214214

215215
Platforms []string `json:"platforms,omitempty"`
216216
ExcludedPlatforms []string `json:"excluded_platforms,omitempty"`
217+
218+
// PatchGlibc applies a function to the package's derivation that
219+
// patches any ELF binaries to use the latest version of nixpkgs#glibc.
220+
PatchGlibc bool `json:"patch_glibc,omitempty"`
217221
}
218222

219223
func NewVersionOnlyPackage(name, version string) Package {

‎internal/devpkg/package.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ type Package struct {
4747
// example: github:nixos/nixpkgs/5233fd2ba76a3accb5aaa999c00509a11fd0793c#hello
4848
Raw string
4949

50+
// PatchGlibc applies a function to the package's derivation that
51+
// patches any ELF binaries to use the latest version of nixpkgs#glibc.
52+
PatchGlibc bool
53+
5054
// isInstallable is true if the package may be enabled on the current platform.
5155
isInstallable bool
5256

@@ -65,8 +69,10 @@ func PackageFromStrings(rawNames []string, l lock.Locker) []*Package {
6569

6670
func PackagesFromConfig(config *devconfig.Config, l lock.Locker) []*Package {
6771
result := []*Package{}
68-
for _, pkg := range config.Packages.Collection {
69-
result = append(result, newPackage(pkg.VersionedName(), pkg.IsEnabledOnPlatform(), l))
72+
for _, cfgPkg := range config.Packages.Collection {
73+
pkg := newPackage(cfgPkg.VersionedName(), cfgPkg.IsEnabledOnPlatform(), l)
74+
pkg.PatchGlibc = cfgPkg.PatchGlibc
75+
result = append(result, pkg)
7076
}
7177
return result
7278
}

‎internal/shellgen/flake_input.go

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -47,28 +47,33 @@ func (f *flakeInput) PkgImportName() string {
4747
return f.Name + "-pkgs"
4848
}
4949

50-
func (f *flakeInput) BuildInputs() ([]string, error) {
51-
var err error
52-
attributePaths := lo.Map(f.Packages, func(pkg *devpkg.Package, _ int) string {
53-
attributePath, attributePathErr := pkg.FullPackageAttributePath()
54-
if attributePathErr != nil {
55-
err = attributePathErr
56-
}
57-
return attributePath
58-
})
59-
if err != nil {
60-
return nil, err
50+
type buildInput struct {
51+
AttrPath string
52+
PatchGlibc bool
53+
}
54+
55+
func (f *flakeInput) BuildInputs() ([]buildInput, error) {
56+
inputs := make([]buildInput, len(f.Packages))
57+
prefix := f.Name
58+
if f.IsNixpkgs() {
59+
prefix = f.PkgImportName()
6160
}
62-
if !f.IsNixpkgs() {
63-
return lo.Map(attributePaths, func(pkg string, _ int) string {
64-
return f.Name + "." + pkg
65-
}), nil
61+
prefix += "."
62+
for i, pkg := range f.Packages {
63+
attrPath, err := pkg.FullPackageAttributePath()
64+
if err != nil {
65+
return nil, err
66+
}
67+
if f.IsNixpkgs() {
68+
// Remove the legacyPackages.<system> prefix.
69+
attrPath = strings.SplitN(attrPath, ".", 3)[2]
70+
}
71+
inputs[i] = buildInput{
72+
AttrPath: prefix + attrPath,
73+
PatchGlibc: pkg.PatchGlibc,
74+
}
6675
}
67-
return lo.Map(attributePaths, func(pkg string, _ int) string {
68-
parts := strings.Split(pkg, ".")
69-
// Ugh, not sure if this is reliable?
70-
return f.PkgImportName() + "." + strings.Join(parts[2:], ".")
71-
}), nil
76+
return inputs, nil
7277
}
7378

7479
// flakeInputs returns a list of flake inputs for the top level flake.nix

‎internal/shellgen/flake_plan.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package shellgen
33
import (
44
"context"
55
"runtime/trace"
6+
"slices"
67

78
"go.jetpack.io/devbox/internal/devpkg"
89
"go.jetpack.io/devbox/internal/nix"
@@ -65,3 +66,9 @@ func newFlakePlan(ctx context.Context, devbox devboxer) (*flakePlan, error) {
6566
System: nix.System(),
6667
}, nil
6768
}
69+
70+
func (f flakePlan) PatchGlibc() bool {
71+
return slices.ContainsFunc(f.Packages, func(p *devpkg.Package) bool {
72+
return p.PatchGlibc
73+
})
74+
}

‎internal/shellgen/generate.go

Lines changed: 78 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@
44
package shellgen
55

66
import (
7+
"bufio"
78
"bytes"
89
"context"
910
"embed"
10-
"io"
1111
"io/fs"
1212
"os"
1313
"os/exec"
@@ -20,6 +20,7 @@ import (
2020
"go.jetpack.io/devbox/internal/boxcli/featureflag"
2121
"go.jetpack.io/devbox/internal/cuecfg"
2222
"go.jetpack.io/devbox/internal/debug"
23+
"go.jetpack.io/devbox/internal/redact"
2324
)
2425

2526
//go:embed tmpl/*
@@ -61,10 +62,7 @@ func GenerateForPrintEnv(ctx context.Context, devbox devboxer) error {
6162
// Cache and buffers for generating templated files.
6263
var (
6364
tmplCache = map[string]*template.Template{}
64-
65-
// Most generated files are < 4KiB.
66-
tmplNewBuf = bytes.NewBuffer(make([]byte, 0, 4096))
67-
tmplOldBuf = bytes.NewBuffer(make([]byte, 0, 4096))
65+
tmplBuf bytes.Buffer
6866
)
6967

7068
func writeFromTemplate(path string, plan any, tmplName, generatedName string) error {
@@ -75,61 +73,92 @@ func writeFromTemplate(path string, plan any, tmplName, generatedName string) er
7573
tmpl.Funcs(templateFuncs)
7674

7775
var err error
78-
tmpl, err = tmpl.ParseFS(tmplFS, "tmpl/"+tmplKey)
76+
glob := "tmpl/" + tmplKey
77+
tmpl, err = tmpl.ParseFS(tmplFS, glob)
7978
if err != nil {
80-
return errors.WithStack(err)
79+
return redact.Errorf("parse embedded tmplFS glob %q: %v", redact.Safe(glob), redact.Safe(err))
8180
}
8281
tmplCache[tmplKey] = tmpl
8382
}
84-
tmplNewBuf.Reset()
85-
if err := tmpl.Execute(tmplNewBuf, plan); err != nil {
86-
return errors.WithStack(err)
83+
tmplBuf.Reset()
84+
if err := tmpl.Execute(&tmplBuf, plan); err != nil {
85+
return redact.Errorf("execute template %s: %v", redact.Safe(tmplKey), err)
8786
}
8887

8988
// In some circumstances, Nix looks at the mod time of a file when
9089
// caching, so we only want to update the file if something has
9190
// changed. Blindly overwriting the file could invalidate Nix's cache
9291
// every time, slowing down evaluation considerably.
93-
var (
94-
outPath = filepath.Join(path, generatedName)
95-
flag = os.O_RDWR | os.O_CREATE
96-
perm = fs.FileMode(0o644)
97-
)
98-
outFile, err := os.OpenFile(outPath, flag, perm)
99-
if errors.Is(err, fs.ErrNotExist) {
100-
if err := os.MkdirAll(path, 0o755); err != nil {
101-
return errors.WithStack(err)
102-
}
103-
outFile, err = os.OpenFile(outPath, flag, perm)
92+
err := overwriteFileIfChanged(filepath.Join(path, generatedName), tmplBuf.Bytes(), 0o644)
93+
if err != nil {
94+
return redact.Errorf("write %s to file: %v", redact.Safe(tmplName), err)
10495
}
96+
return nil
97+
}
98+
99+
// writeGlibcPatchScript writes the embedded glibc patching script to disk so
100+
// that a generated flake can use it.
101+
func writeGlibcPatchScript(path string) error {
102+
script, err := fs.ReadFile(tmplFS, "tmpl/glibc-patch.bash")
105103
if err != nil {
106-
return errors.WithStack(err)
104+
return redact.Errorf("read embedded glibc-patch.bash: %v", redact.Safe(err))
105+
}
106+
err = overwriteFileIfChanged(path, script, 0o755)
107+
if err != nil {
108+
return redact.Errorf("write glibc-patch.bash to file: %v", err)
107109
}
108-
defer outFile.Close()
110+
return nil
111+
}
112+
113+
// overwriteFileIfChanged checks that the contents of f == data, and overwrites
114+
// f if they differ. It also ensures that f's permissions are set to perm.
115+
func overwriteFileIfChanged(path string, data []byte, perm os.FileMode) error {
116+
flag := os.O_RDWR | os.O_CREATE
117+
file, err := os.OpenFile(path, flag, perm)
118+
if errors.Is(err, os.ErrNotExist) {
119+
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
120+
return err
121+
}
109122

110-
// Only read len(tmplWriteBuf) + 1 from the existing file so we can
111-
// check if the lengths are different without reading the whole thing.
112-
tmplOldBuf.Reset()
113-
tmplOldBuf.Grow(tmplNewBuf.Len() + 1)
114-
_, err = io.Copy(tmplOldBuf, io.LimitReader(outFile, int64(tmplNewBuf.Len())+1))
123+
// Definitely a new file if we had to make the directory.
124+
return os.WriteFile(path, data, perm)
125+
}
115126
if err != nil {
116-
return errors.WithStack(err)
127+
return err
117128
}
118-
if bytes.Equal(tmplNewBuf.Bytes(), tmplOldBuf.Bytes()) {
119-
return nil
129+
defer file.Close()
130+
131+
fi, err := file.Stat()
132+
if err != nil || fi.Mode().Perm() != perm {
133+
if err := file.Chmod(perm); err != nil {
134+
return err
135+
}
120136
}
121137

122-
// Replace the existing file contents.
123-
if _, err := outFile.Seek(0, io.SeekStart); err != nil {
124-
return errors.WithStack(err)
138+
// Fast path - check if the lengths differ.
139+
if err == nil && fi.Size() != int64(len(data)) {
140+
return overwriteFile(file, data, 0)
125141
}
126-
if err := outFile.Truncate(int64(tmplNewBuf.Len())); err != nil {
127-
return errors.WithStack(err)
142+
143+
r := bufio.NewReader(file)
144+
for offset := range data {
145+
b, err := r.ReadByte()
146+
if err != nil || b != data[offset] {
147+
return overwriteFile(file, data, offset)
148+
}
128149
}
129-
if _, err := io.Copy(outFile, tmplNewBuf); err != nil {
130-
return errors.WithStack(err)
150+
return nil
151+
}
152+
153+
// overwriteFile truncates f to len(data) and writes data[offset:] beginning at
154+
// the same offset in f.
155+
func overwriteFile(f *os.File, data []byte, offset int) error {
156+
err := f.Truncate(int64(len(data)))
157+
if err != nil {
158+
return err
131159
}
132-
return errors.WithStack(outFile.Close())
160+
_, err = f.WriteAt(data[offset:], int64(offset))
161+
return err
133162
}
134163

135164
func toJSON(a any) string {
@@ -157,6 +186,13 @@ func makeFlakeFile(d devboxer, outPath string, plan *flakePlan) error {
157186
return errors.WithStack(err)
158187
}
159188

189+
if plan.PatchGlibc() {
190+
err := writeGlibcPatchScript(filepath.Join(flakeDir, "glibc-patch.bash"))
191+
if err != nil {
192+
return err
193+
}
194+
}
195+
160196
if !isProjectInGitRepo(outPath) {
161197
// if we are not in a git repository, then carry on
162198
return nil
@@ -178,8 +214,9 @@ func makeFlakeFile(d devboxer, outPath string, plan *flakePlan) error {
178214
return errors.WithStack(err)
179215
}
180216

181-
// add the flake.nix file to git
182-
cmd = exec.Command("git", "-C", flakeDir, "add", "flake.nix")
217+
// Any files that flake.nix needs at build time must be in git.
218+
// Otherwise, Nix won't copy it into the flake's build environment.
219+
cmd = exec.Command("git", "-C", flakeDir, "add", ".")
183220
if debug.IsEnabled() {
184221
cmd.Stdin = os.Stdin
185222
cmd.Stdout = os.Stdout

‎internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,14 @@
33

44
inputs = {
55
nixpkgs.url = "{{ .NixpkgsInfo.URL }}";
6+
67
{{- range .FlakeInputs }}
78
{{.Name}}.url = "{{.URLWithCaching}}";
89
{{- end }}
10+
11+
{{- if .PatchGlibc }}
12+
nixpkgs-unstable.url = "nixpkgs";
13+
{{- end }}
914
};
1015

1116
outputs = {
@@ -14,6 +19,9 @@
1419
{{- range .FlakeInputs }}
1520
{{.Name}},
1621
{{- end }}
22+
{{- if .PatchGlibc }}
23+
nixpkgs-unstable,
24+
{{- end }}
1725
}:
1826
let
1927
pkgs = nixpkgs.legacyPackages.{{ .System }};
@@ -29,25 +37,59 @@
2937
{{- end }}
3038
{{- end }}
3139
];
40+
41+
overlays = [
42+
{{- range $flake.Packages }}
43+
{{- if .PatchGlibc }}
44+
(final: prev: {
45+
{{.PackageAttributePath}} = prev.{{.PackageAttributePath}};
46+
})
47+
{{- end }}
48+
{{- end }}
49+
];
3250
});
3351
{{- end }}
3452
{{- end }}
53+
54+
{{ if .PatchGlibc -}}
55+
patchGlibc = pkg: derivation {
56+
name = "devbox-patched-glibc";
57+
system = "{{.System}}";
58+
59+
# Set these attributes so that glibc-patch.bash has access to their
60+
# paths via environment variables of the same name.
61+
inherit pkg;
62+
glibc = nixpkgs.legacyPackages."{{.System}}".glibc;
63+
coreutils = nixpkgs.legacyPackages."{{.System}}".coreutils;
64+
file = nixpkgs.legacyPackages."{{.System}}".file;
65+
findutils = nixpkgs.legacyPackages."{{.System}}".findutils;
66+
patchelf = nixpkgs.legacyPackages."{{.System}}".patchelf;
67+
ripgrep = nixpkgs.legacyPackages."{{.System}}".ripgrep;
68+
69+
builder = "${nixpkgs-unstable.legacyPackages.{{.System}}.bash}/bin/bash";
70+
args = [ ./glibc-patch.bash ];
71+
};
72+
{{ end }}
3573
in
3674
{
3775
devShells.{{ .System }}.default = pkgs.mkShell {
3876
buildInputs = [
3977
{{- range .Packages }}
40-
{{- if .IsInBinaryCache }}
41-
(builtins.fetchClosure{
78+
{{ if .IsInBinaryCache -}}
79+
{{ if .PatchGlibc -}} (patchGlibc {{ end -}}
80+
(builtins.fetchClosure {
4281
fromStore = "{{ $.BinaryCache }}";
4382
fromPath = "{{ .InputAddressedPath }}";
4483
inputAddressed = true;
4584
})
85+
{{- if .PatchGlibc -}} ) {{- end -}}
4686
{{- end }}
4787
{{- end }}
4888
{{- range .FlakeInputs }}
4989
{{- range .BuildInputs }}
50-
{{.}}
90+
{{ if .PatchGlibc -}} (patchGlibc {{ end -}}
91+
{{.AttrPath}}
92+
{{- if .PatchGlibc -}} ) {{- end -}}
5193
{{- end }}
5294
{{- end }}
5395
];

0 commit comments

Comments
 (0)