|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "bytes" |
| 5 | + "context" |
| 6 | + "encoding/json" |
| 7 | + "flag" |
| 8 | + "fmt" |
| 9 | + "io/fs" |
| 10 | + "os" |
| 11 | + "os/exec" |
| 12 | + "os/signal" |
| 13 | + "path/filepath" |
| 14 | + "regexp" |
| 15 | + "sort" |
| 16 | + "strings" |
| 17 | + "text/tabwriter" |
| 18 | + "time" |
| 19 | + |
| 20 | + "github.com/sirupsen/logrus" |
| 21 | +) |
| 22 | + |
| 23 | +type mode string |
| 24 | + |
| 25 | +const ( |
| 26 | + summarize mode = "summarize" |
| 27 | + synchronize mode = "synchronize" |
| 28 | +) |
| 29 | + |
| 30 | +type options struct { |
| 31 | + stagingDir string |
| 32 | + commitFileOutput string |
| 33 | + commitFileInput string |
| 34 | + mode string |
| 35 | + logLevel string |
| 36 | +} |
| 37 | + |
| 38 | +func (o *options) Bind(fs *flag.FlagSet) { |
| 39 | + fs.StringVar(&o.stagingDir, "staging-dir", "staging/", "Directory for staging repositories.") |
| 40 | + fs.StringVar(&o.mode, "mode", string(summarize), "Operation mode.") |
| 41 | + fs.StringVar(&o.commitFileOutput, "commits-output", "", "File to write commits data to after resolving what needs to be synced.") |
| 42 | + fs.StringVar(&o.commitFileInput, "commits-input", "", "File to read commits data from in order to drive sync process.") |
| 43 | + fs.StringVar(&o.logLevel, "log-level", logrus.InfoLevel.String(), "Logging level.") |
| 44 | +} |
| 45 | + |
| 46 | +func (o *options) Validate() error { |
| 47 | + switch mode(o.mode) { |
| 48 | + case summarize, synchronize: |
| 49 | + default: |
| 50 | + return fmt.Errorf("--mode must be one of %v", []mode{summarize, synchronize}) |
| 51 | + } |
| 52 | + |
| 53 | + if _, err := logrus.ParseLevel(o.logLevel); err != nil { |
| 54 | + return fmt.Errorf("--log-level invalid: %w", err) |
| 55 | + } |
| 56 | + return nil |
| 57 | +} |
| 58 | + |
| 59 | +func main() { |
| 60 | + logger := logrus.New() |
| 61 | + opts := options{} |
| 62 | + opts.Bind(flag.CommandLine) |
| 63 | + flag.Parse() |
| 64 | + |
| 65 | + if err := opts.Validate(); err != nil { |
| 66 | + logger.WithError(err).Fatal("invalid options") |
| 67 | + } |
| 68 | + |
| 69 | + logLevel, _ := logrus.ParseLevel(opts.logLevel) |
| 70 | + logger.SetLevel(logLevel) |
| 71 | + |
| 72 | + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) |
| 73 | + defer cancel() |
| 74 | + |
| 75 | + var commits []commit |
| 76 | + var err error |
| 77 | + if opts.commitFileInput != "" { |
| 78 | + rawCommits, err := os.ReadFile(opts.commitFileInput) |
| 79 | + if err != nil { |
| 80 | + logrus.WithError(err).Fatal("could not read input file") |
| 81 | + } |
| 82 | + if err := json.Unmarshal(rawCommits, &commits); err != nil { |
| 83 | + logrus.WithError(err).Fatal("could not unmarshal input commits") |
| 84 | + } |
| 85 | + } else { |
| 86 | + commits, err = detectNewCommits(ctx, logger.WithField("phase", "detect"), opts.stagingDir) |
| 87 | + if err != nil { |
| 88 | + logger.WithError(err).Fatal("failed to detect commits") |
| 89 | + } |
| 90 | + } |
| 91 | + |
| 92 | + if opts.commitFileOutput != "" { |
| 93 | + commitsJson, err := json.Marshal(commits) |
| 94 | + if err != nil { |
| 95 | + logrus.WithError(err).Fatal("could not marshal commits") |
| 96 | + } |
| 97 | + if err := os.WriteFile(opts.commitFileOutput, commitsJson, 0666); err != nil { |
| 98 | + logrus.WithError(err).Fatal("could not write commits") |
| 99 | + } |
| 100 | + } |
| 101 | + |
| 102 | + switch mode(opts.mode) { |
| 103 | + case summarize: |
| 104 | + writer := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) |
| 105 | + for _, commit := range commits { |
| 106 | + if _, err := fmt.Fprintln(writer, commit.Date.Format(time.DateTime)+"\t"+"operator-framework/"+commit.Repo+"\t", commit.Hash+"\t"+commit.Author+"\t"+commit.Message); err != nil { |
| 107 | + logger.WithError(err).Error("failed to write output") |
| 108 | + } |
| 109 | + } |
| 110 | + if err := writer.Flush(); err != nil { |
| 111 | + logger.WithError(err).Error("failed to flush output") |
| 112 | + } |
| 113 | + case synchronize: |
| 114 | + for i, commit := range commits { |
| 115 | + commitLogger := logger.WithField("commit", commit.Hash) |
| 116 | + commitLogger.Infof("cherry-picking commit %d/%d", i+1, len(commits)) |
| 117 | + if err := cherryPick(ctx, commitLogger, commit); err != nil { |
| 118 | + logger.WithError(err).Error("failed to cherry-pick commit") |
| 119 | + break |
| 120 | + } |
| 121 | + } |
| 122 | + } |
| 123 | +} |
| 124 | + |
| 125 | +type commit struct { |
| 126 | + Date time.Time `json:"date"` |
| 127 | + Hash string `json:"hash,omitempty"` |
| 128 | + Author string `json:"author,omitempty"` |
| 129 | + Message string `json:"message,omitempty"` |
| 130 | + Repo string `json:"repo,omitempty"` |
| 131 | +} |
| 132 | + |
| 133 | +var repoRegex = regexp.MustCompile(`Upstream-repository: ([^ ]+)\n`) |
| 134 | +var commitRegex = regexp.MustCompile(`Upstream-commit: ([a-f0-9]+)\n`) |
| 135 | + |
| 136 | +func detectNewCommits(ctx context.Context, logger *logrus.Entry, stagingDir string) ([]commit, error) { |
| 137 | + lastCommits := map[string]string{} |
| 138 | + if err := fs.WalkDir(os.DirFS(stagingDir), ".", func(path string, d fs.DirEntry, err error) error { |
| 139 | + if err != nil { |
| 140 | + return err |
| 141 | + } |
| 142 | + if d == nil || !d.IsDir() { |
| 143 | + return nil |
| 144 | + } |
| 145 | + |
| 146 | + if path == "." { |
| 147 | + return nil |
| 148 | + } |
| 149 | + logger = logger.WithField("repo", path) |
| 150 | + logger.Debug("detecting commits") |
| 151 | + output, err := runCommand(logger, exec.CommandContext(ctx, |
| 152 | + "git", "log", |
| 153 | + "-n", "1", |
| 154 | + "--grep", "Upstream-repository: "+path, |
| 155 | + "--grep", "Upstream-commit", |
| 156 | + "--all-match", |
| 157 | + "--pretty=%B", |
| 158 | + "--", |
| 159 | + filepath.Join(stagingDir, path), |
| 160 | + )) |
| 161 | + if err != nil { |
| 162 | + return err |
| 163 | + } |
| 164 | + var lastCommit string |
| 165 | + commitMatches := commitRegex.FindStringSubmatch(output) |
| 166 | + if len(commitMatches) > 0 { |
| 167 | + if len(commitMatches[0]) > 1 { |
| 168 | + lastCommit = string(commitMatches[1]) |
| 169 | + } |
| 170 | + } |
| 171 | + if lastCommit != "" { |
| 172 | + logger.WithField("commit", lastCommit).Debug("found last commit synchronized with staging") |
| 173 | + lastCommits[path] = lastCommit |
| 174 | + } |
| 175 | + |
| 176 | + if path != "." { |
| 177 | + return fs.SkipDir |
| 178 | + } |
| 179 | + return nil |
| 180 | + }); err != nil { |
| 181 | + return nil, fmt.Errorf("failed to walk %s: %w", stagingDir, err) |
| 182 | + } |
| 183 | + |
| 184 | + var commits []commit |
| 185 | + for repo, lastCommit := range lastCommits { |
| 186 | + if _, err := runCommand(logger, exec.CommandContext(ctx, |
| 187 | + "git", "fetch", |
| 188 | + "[email protected]:operator-framework/"+repo, |
| 189 | + "master", |
| 190 | + )); err != nil { |
| 191 | + return nil, err |
| 192 | + } |
| 193 | + |
| 194 | + output, err := runCommand(logger, exec.CommandContext(ctx, |
| 195 | + "git", "log", |
| 196 | + "--pretty=%H", |
| 197 | + lastCommit+"...FETCH_HEAD", |
| 198 | + )) |
| 199 | + if err != nil { |
| 200 | + return nil, err |
| 201 | + } |
| 202 | + |
| 203 | + for _, line := range strings.Split(output, "\n") { |
| 204 | + line = strings.TrimSpace(line) |
| 205 | + if line != "" { |
| 206 | + infoCmd := exec.CommandContext(ctx, |
| 207 | + "git", "show", |
| 208 | + line, |
| 209 | + "--pretty=format:%H\u00A0%cI\u00A0%an\u00A0%s", |
| 210 | + "--quiet", |
| 211 | + ) |
| 212 | + stdout, stderr := bytes.Buffer{}, bytes.Buffer{} |
| 213 | + infoCmd.Stdout = &stdout |
| 214 | + infoCmd.Stderr = &stderr |
| 215 | + logger.WithField("command", infoCmd.String()).Debug("running command") |
| 216 | + if err := infoCmd.Run(); err != nil { |
| 217 | + return nil, fmt.Errorf("failed to run command: %s %s: %w", stdout.String(), stderr.String(), err) |
| 218 | + } |
| 219 | + parts := strings.Split(stdout.String(), "\u00A0") |
| 220 | + if len(parts) != 4 { |
| 221 | + return nil, fmt.Errorf("incorrect parts from git output: %v", stdout.String()) |
| 222 | + } |
| 223 | + committedTime, err := time.Parse(time.RFC3339, parts[1]) |
| 224 | + if err != nil { |
| 225 | + return nil, fmt.Errorf("invalid time %s: %w", parts[1], err) |
| 226 | + } |
| 227 | + commits = append(commits, commit{ |
| 228 | + Hash: parts[0], |
| 229 | + Date: committedTime, |
| 230 | + Author: parts[2], |
| 231 | + Message: parts[3], |
| 232 | + Repo: repo, |
| 233 | + }) |
| 234 | + } |
| 235 | + } |
| 236 | + } |
| 237 | + sort.Slice(commits, func(i, j int) bool { |
| 238 | + return commits[i].Date.Before(commits[j].Date) |
| 239 | + }) |
| 240 | + return commits, nil |
| 241 | +} |
| 242 | + |
| 243 | +func cherryPick(ctx context.Context, logger *logrus.Entry, c commit) error { |
| 244 | + { |
| 245 | + output, err := runCommand(logger, exec.CommandContext(ctx, |
| 246 | + "git", "cherry-pick", |
| 247 | + "--allow-empty", "--keep-redundant-commits", |
| 248 | + "-Xsubtree=staging/"+c.Repo, c.Hash, |
| 249 | + )) |
| 250 | + if err != nil { |
| 251 | + if strings.Contains(output, "vendor/modules.txt deleted in HEAD and modified in") { |
| 252 | + // we remove vendor directories for everything under staging/, but some of the upstream repos have them |
| 253 | + if _, err := runCommand(logger, exec.CommandContext(ctx, |
| 254 | + "git", "rm", "--cached", "-r", "--ignore-unmatch", "staging/"+c.Repo+"/vendor", |
| 255 | + )); err != nil { |
| 256 | + return err |
| 257 | + } |
| 258 | + if _, err := runCommand(logger, exec.CommandContext(ctx, |
| 259 | + "git", "cherry-pick", "--continue", |
| 260 | + )); err != nil { |
| 261 | + return err |
| 262 | + } |
| 263 | + } else { |
| 264 | + return err |
| 265 | + } |
| 266 | + } |
| 267 | + } |
| 268 | + |
| 269 | + for _, cmd := range []*exec.Cmd{ |
| 270 | + withEnv(exec.CommandContext(ctx, |
| 271 | + "go", "mod", "tidy", |
| 272 | + ), os.Environ()...), |
| 273 | + withEnv(exec.CommandContext(ctx, |
| 274 | + "go", "mod", "vendor", |
| 275 | + ), os.Environ()...), |
| 276 | + withEnv(exec.CommandContext(ctx, |
| 277 | + "go", "mod", "verify", |
| 278 | + ), os.Environ()...), |
| 279 | + withEnv(exec.CommandContext(ctx, |
| 280 | + "make", "generate-manifests", |
| 281 | + ), os.Environ()...), |
| 282 | + exec.CommandContext(ctx, |
| 283 | + "git", "add", |
| 284 | + "staging/"+c.Repo, |
| 285 | + "vendor", "go.mod", "go.sum", |
| 286 | + "manifests", "pkg/manifests", |
| 287 | + ), |
| 288 | + exec.CommandContext(ctx, |
| 289 | + "git", "commit", |
| 290 | + "--amend", "--allow-empty", "--no-edit", |
| 291 | + "--trailer", "Upstream-repository: "+c.Repo, |
| 292 | + "--trailer", "Upstream-commit: "+c.Hash, |
| 293 | + "staging/"+c.Repo, |
| 294 | + "vendor", "go.mod", "go.sum", |
| 295 | + "manifests", "pkg/manifests", |
| 296 | + ), |
| 297 | + } { |
| 298 | + if _, err := runCommand(logger, cmd); err != nil { |
| 299 | + return err |
| 300 | + } |
| 301 | + } |
| 302 | + |
| 303 | + return nil |
| 304 | +} |
| 305 | + |
| 306 | +func runCommand(logger *logrus.Entry, cmd *exec.Cmd) (string, error) { |
| 307 | + output := bytes.Buffer{} |
| 308 | + cmd.Stdout = &output |
| 309 | + cmd.Stderr = &output |
| 310 | + logger.WithField("command", cmd.String()).Debug("running command") |
| 311 | + if err := cmd.Run(); err != nil { |
| 312 | + return output.String(), fmt.Errorf("failed to run command: %s: %w", output.String(), err) |
| 313 | + } |
| 314 | + return output.String(), nil |
| 315 | +} |
| 316 | + |
| 317 | +func withEnv(command *exec.Cmd, env ...string) *exec.Cmd { |
| 318 | + command.Env = append(command.Env, env...) |
| 319 | + return command |
| 320 | +} |
0 commit comments