Auto merge of #2469 - jtgeibel:fix-dev-git-index-endpoint, r=JohnTitor

Fix serving `/git/index` during local development

The "index checkout" directory is used to initialize the bare index repo
but is not kept in sync after that. It seems unnecessary to require all
environments to define the `GIT_REPO_CHECKOUT` variable, so it is
dropped as well.

During local development, the `tmp/index-bare` repo is automatically
served at `/git/index` by the HTTP backend.

r? @JohnTitor

Author: bors

.env.sample

@@ -25,11 +25,10 @@ export TEST_DATABASE_URL=
 # not needed if the S3 bucket is in US standard
 # export S3_REGION=
 
-# Remote and local locations of the registry index. You can leave these to
-# use a `tmp` subdirectory of the working directory, which is what the
-# script in `./script/init-local-index.sh` will set up for you.
+# Upstream location of the registry index. Background jobs will push to
+# this URL. The default points to a local index for development.
+# Run `./script/init-local-index.sh` to initialize this repo.
 export GIT_REPO_URL=file://$PWD/tmp/index-bare
-export GIT_REPO_CHECKOUT=./tmp/index-co
 
 # Credentials for talking to github. You can leave these blank if you're
 # not logging into your crates.io instance.

app.json

@@ -42,8 +42,7 @@
           "generator": "secret"
       },
       "HEROKU": "1",
-      "MIRROR": "1",
-      "GIT_REPO_CHECKOUT": "./tmp/index-co"
+      "MIRROR": "1"
   },
   "formation": {
       "web": {

docker-compose.yml

@@ -18,7 +18,6 @@ services:
       DATABASE_URL: postgres://postgres:password@postgres/cargo_registry
       SESSION_KEY: badkeyabcdefghijklmnopqrstuvwxyzabcdef
       GIT_REPO_URL: file://./tmp/index-bare
-      GIT_REPO_CHECKOUT: ./tmp/index-co
       GH_CLIENT_ID: ""
       GH_CLIENT_SECRET: ""
     links:

script/init-local-index.sh

@@ -8,14 +8,14 @@ if [ -d tmp/index-bare ]; then
 fi
 
 mkdir -p tmp
-rm -rf tmp/index-bare tmp/index-co
+rm -rf tmp/index-bare tmp/index-tmp
 
 echo "Initializing repository in tmp/index-bare..."
 git init -q --bare tmp/index-bare
 
-echo "Creating checkout in tmp/index-bare..."
-git init -q tmp/index-co
-cd tmp/index-co
+echo "Creating temporary clone in tmp/index-tmp..."
+git init -q tmp/index-tmp
+cd tmp/index-tmp
 cat > config.json <<-EOF
 {
   "dl": "http://localhost:8888/api/v1/crates",
@@ -27,7 +27,12 @@ git commit -qm 'Initial commit'
 git remote add origin file://`pwd`/../index-bare
 git push -q origin master -u > /dev/null
 cd ../..
-touch tmp/index-co/.git/git-daemon-export-ok
+
+# Remove the temporary checkout
+rm -rf tmp/index-tmp
+
+# Allow the index to be exported via HTTP during local development
+touch tmp/index-bare/git-daemon-export-ok
 
 cat - <<-EOF
 Your local git index is ready to go!

src/app.rs

@@ -1,7 +1,7 @@
 //! Application-wide components in a struct accessible from each request
 
 use crate::{db, Config, Env};
-use std::{path::PathBuf, sync::Arc, time::Duration};
+use std::{sync::Arc, time::Duration};
 
 use diesel::r2d2;
 use oauth2::basic::BasicClient;
@@ -25,10 +25,6 @@ pub struct App {
     /// A unique key used with conduit_cookie to generate cookies
     pub session_key: String,
 
-    /// The location on disk of the checkout of the crate index git repository
-    /// Only used in the development environment.
-    pub git_repo_checkout: PathBuf,
-
     /// The server configuration
     pub config: Config,
 
@@ -130,7 +126,6 @@ impl App {
             read_only_replica_database,
             github,
             session_key: config.session_key.clone(),
-            git_repo_checkout: config.git_repo_checkout.clone(),
             config: config.clone(),
             http_client,
         }

src/config.rs

@@ -1,12 +1,10 @@
 use crate::publish_rate_limit::PublishRateLimit;
 use crate::{env, uploaders::Uploader, Env, Replica};
-use std::path::PathBuf;
 
 #[derive(Clone, Debug)]
 pub struct Config {
     pub uploader: Uploader,
     pub session_key: String,
-    pub git_repo_checkout: PathBuf,
     pub gh_client_id: String,
     pub gh_client_secret: String,
     pub db_url: String,
@@ -30,7 +28,6 @@ impl Default for Config {
     ///
     /// Pulls values from the following environment variables:
     ///
-    /// - `GIT_REPO_CHECKOUT`: The directory where the registry index was cloned.
     /// - `MIRROR`: Is this instance of cargo_registry a mirror of crates.io.
     /// - `HEROKU`: Is this instance of cargo_registry currently running on Heroku.
     /// - `S3_BUCKET`: The S3 bucket used to store crate files. If not present during development,
@@ -46,7 +43,6 @@ impl Default for Config {
     /// - `BLOCKED_TRAFFIC`: A list of headers and environment variables to use for blocking
     ///.  traffic. See the `block_traffic` module for more documentation.
     fn default() -> Config {
-        let checkout = PathBuf::from(env("GIT_REPO_CHECKOUT"));
         let api_protocol = String::from("https");
         let mirror = if dotenv::var("MIRROR").is_ok() {
             Replica::ReadOnlyMirror
@@ -127,7 +123,6 @@ impl Default for Config {
         Config {
             uploader,
             session_key: env("SESSION_KEY"),
-            git_repo_checkout: checkout,
             gh_client_id: env("GH_CLIENT_ID"),
             gh_client_secret: env("GH_CLIENT_SECRET"),
             db_url: env("DATABASE_URL"),

src/middleware/ember_index_rewrite.rs

@@ -40,7 +40,8 @@ impl Handler for EmberIndexRewrite {
     fn call(&self, req: &mut dyn RequestExt) -> HandlerResult {
         let handler = self.handler.as_ref().unwrap();
 
-        if req.path().starts_with("/api") {
+        // The "/git/" prefix is only used in development (when within a docker container)
+        if req.path().starts_with("/api/") || req.path().starts_with("/git/") {
             handler.call(req)
         } else {
             if let Some(client) = &self.fastboot_client {

src/router.rs

@@ -127,7 +127,7 @@ pub fn build_router(app: &App) -> R404 {
     // In production, for crates.io, cargo gets the index from
     // https://github.com/rust-lang/crates.io-index directly.
     if app.config.env == Env::Development {
-        let s = conduit_git_http_backend::Serve(app.git_repo_checkout.clone());
+        let s = conduit_git_http_backend::Serve("./tmp/index-bare".into());
         let s = Arc::new(s);
         router.get("/git/index/*path", R(Arc::clone(&s)));
         router.post("/git/index/*path", R(s));

src/tests/all.rs

@@ -131,7 +131,6 @@ fn simple_config() -> Config {
     Config {
         uploader,
         session_key: "test this has to be over 32 bytes long".to_string(),
-        git_repo_checkout: git::checkout(),
         gh_client_id: dotenv::var("GH_CLIENT_ID").unwrap_or_default(),
         gh_client_secret: dotenv::var("GH_CLIENT_SECRET").unwrap_or_default(),
         db_url: env("TEST_DATABASE_URL"),

src/tests/git.rs

@@ -12,16 +12,12 @@ fn root() -> PathBuf {
         .join(thread::current().name().unwrap())
 }
 
-pub fn checkout() -> PathBuf {
-    root().join("checkout")
-}
 pub fn bare() -> PathBuf {
     root().join("bare")
 }
 
 pub fn init() {
     static INIT: Once = Once::new();
-    let _ = fs::remove_dir_all(&checkout());
     let _ = fs::remove_dir_all(&bare());
 
     INIT.call_once(|| {