Auto merge of #3592 - jtgeibel:squash-index, r=pietroalbini

Add a background job for squashing the index

This adds a background job that squashes the index into a single commit.
The current plan is to manually enqueue this job on a 6 week schedule,
roughly aligning with new `rustc` releases. Before deploying this, will
need to make sure that the SSH key is allowed to do a force push to the
protected master branch.

This job is derived from a [script] that was periodically run by the
cargo team. Relative to the original script, the push of the snapshot
branch is no longer forced. The job will fail if run more than once on
the same day. (If the first attempt fails before pushing a new root
commit upstream, then retries should succeed as long as the snapshot
can be fast-forwarded.)

[script]: rust-lang/crates-io-cargo-teams#47 (comment)

Author: bors

src/bin/enqueue-job.rs

@@ -1,7 +1,7 @@
 #![warn(clippy::all, rust_2018_idioms)]
 
 use anyhow::{anyhow, Result};
-use cargo_registry::{db, env, tasks};
+use cargo_registry::{db, env, git, tasks};
 use diesel::prelude::*;
 use swirl::schema::background_jobs::dsl::*;
 use swirl::Job;
@@ -36,6 +36,7 @@ fn main() -> Result<()> {
             Ok(tasks::dump_db(database_url, target_name).enqueue(&conn)?)
         }
         "daily_db_maintenance" => Ok(tasks::daily_db_maintenance().enqueue(&conn)?),
+        "squash_index" => Ok(git::squash_index().enqueue(&conn)?),
         other => Err(anyhow!("Unrecognized job type `{}`", other)),
     }
 }

src/git.rs

@@ -1,8 +1,11 @@
 use std::collections::HashMap;
 use std::fs::{self, OpenOptions};
+use std::io::prelude::*;
 use std::path::{Path, PathBuf};
+
+use chrono::Utc;
 use swirl::PerformError;
-use tempfile::{Builder, TempDir};
+use tempfile::TempDir;
 use url::Url;
 
 use crate::background_jobs::Environment;
@@ -146,7 +149,7 @@ pub struct Repository {
 
 impl Repository {
     pub fn open(repository_config: &RepositoryConfig) -> Result<Self, PerformError> {
-        let checkout_path = Builder::new().prefix("git").tempdir()?;
+        let checkout_path = tempfile::Builder::new().prefix("git").tempdir()?;
 
         let repository = git2::build::RepoBuilder::new()
             .fetch_options(Self::fetch_options(&repository_config.credentials))
@@ -203,12 +206,11 @@ impl Repository {
         self.repository
             .commit(Some("HEAD"), &sig, &sig, msg, &tree, &[&parent])?;
 
-        self.push()
+        self.push("refs/heads/master")
     }
 
-    /// Push the current branch to "refs/heads/master"
-    fn push(&self) -> Result<(), PerformError> {
-        let refname = "refs/heads/master";
+    /// Push the current branch to the provided refname
+    fn push(&self, refspec: &str) -> Result<(), PerformError> {
         let mut ref_status = Ok(());
         let mut callback_called = false;
         {
@@ -217,8 +219,7 @@ impl Repository {
             callbacks.credentials(|_, user_from_url, cred_type| {
                 self.credentials.git2_callback(user_from_url, cred_type)
             });
-            callbacks.push_update_reference(|cb_refname, status| {
-                assert_eq!(refname, cb_refname);
+            callbacks.push_update_reference(|_, status| {
                 if let Some(s) = status {
                     ref_status = Err(format!("failed to push a ref: {}", s).into())
                 }
@@ -227,7 +228,7 @@ impl Repository {
             });
             let mut opts = git2::PushOptions::new();
             opts.remote_callbacks(callbacks);
-            origin.push(&[refname], Some(&mut opts))?;
+            origin.push(&[refspec], Some(&mut opts))?;
         }
 
         if !callback_called {
@@ -276,6 +277,24 @@ impl Repository {
         opts.remote_callbacks(callbacks);
         opts
     }
+
+    /// Reset `HEAD` to a single commit with all the index contents, but no parent
+    fn squash_to_single_commit(&self, msg: &str) -> Result<(), PerformError> {
+        let tree = self.repository.find_commit(self.head_oid()?)?.tree()?;
+        let sig = self.repository.signature()?;
+
+        // We cannot update an existing `update_ref`, because that requires the
+        // first parent of this commit to match the ref's current value.
+        // Instead, create the commit and then do a hard reset.
+        let commit = self.repository.commit(None, &sig, &sig, msg, &tree, &[])?;
+        let commit = self
+            .repository
+            .find_object(commit, Some(git2::ObjectType::Commit))?;
+        self.repository
+            .reset(&commit, git2::ResetType::Hard, None)?;
+
+        Ok(())
+    }
 }
 
 #[swirl::background_job]
@@ -357,3 +376,77 @@ pub fn yank(
         Ok(())
     })
 }
+
+/// Collapse the index into a single commit, archiving the current history in a snapshot branch.
+#[swirl::background_job]
+pub fn squash_index(env: &Environment) -> Result<(), PerformError> {
+    let repo = env.lock_index()?;
+    println!("Squashing the index into a single commit.");
+
+    let now = Utc::now().format("%Y-%m-%d");
+    let original_head = repo.head_oid()?.to_string();
+    let msg = format!("Collapse index into one commit\n\n\
+
+        Previous HEAD was {}, now on the `snapshot-{}` branch\n\n\
+
+        More information about this change can be found [online] and on [this issue].\n\n\
+
+        [online]: https://internals.rust-lang.org/t/cargos-crate-index-upcoming-squash-into-one-commit/8440\n\
+        [this issue]: https://github.com/rust-lang/crates-io-cargo-teams/issues/47", original_head, now);
+
+    repo.squash_to_single_commit(&msg)?;
+
+    // Shell out to git because libgit2 does not currently support push leases
+
+    let key = match &repo.credentials {
+        Credentials::Ssh { key } => key,
+        Credentials::Http { .. } => {
+            return Err(String::from("squash_index: Password auth not supported").into())
+        }
+        _ => return Err(String::from("squash_index: Could not determine credentials").into()),
+    };
+
+    // When running on production, ensure the file is created in tmpfs and not persisted to disk
+    #[cfg(target_os = "linux")]
+    let mut temp_key_file = tempfile::Builder::new().tempfile_in("/dev/shm")?;
+
+    // For other platforms, default to std::env::tempdir()
+    #[cfg(not(target_os = "linux"))]
+    let mut temp_key_file = tempfile::Builder::new().tempfile()?;
+
+    temp_key_file.write_all(key.as_bytes())?;
+
+    let checkout_path = repo.checkout_path.path();
+    let output = std::process::Command::new("git")
+        .current_dir(checkout_path)
+        .env(
+            "GIT_SSH_COMMAND",
+            format!(
+                "ssh -o StrictHostKeyChecking=accept-new -i {}",
+                temp_key_file.path().display()
+            ),
+        )
+        .args(&[
+            "push",
+            // Both updates should succeed or fail together
+            "--atomic",
+            "origin",
+            // Overwrite master, but only if it server matches the expected value
+            &format!("--force-with-lease=refs/heads/master:{}", original_head),
+            // The new squashed commit is pushed to master
+            "HEAD:refs/heads/master",
+            // The previous value of HEAD is pushed to a snapshot branch
+            &format!("{}:refs/heads/snapshot-{}", original_head, now),
+        ])
+        .output()?;
+
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        let message = format!("Running git command failed with: {}", stderr);
+        return Err(message.into());
+    }
+
+    println!("The index has been successfully squashed.");
+
+    Ok(())
+}