Rewrite record/replay with Hyper

We're sending bodies now with actual tarballs, which means that we need any
amount of functionality over text parsing. Let's just use a real HTTP library
instead of janky line-by-line parsing.

Author: alexcrichton

Cargo.lock

@@ -75,6 +75,11 @@ civet = "0.9"
 [dev-dependencies]
 conduit-test = "0.8"
 bufstream = "0.1"
+hyper = "0.11"
+hyper-tls = "0.1"
+futures = "0.1"
+tokio-core = "0.1"
+tokio-service = "0.1"
 
 [build-dependencies]
 dotenv = "0.10"

Cargo.toml

@@ -20,6 +20,8 @@ extern crate serde_json;
 extern crate time;
 extern crate url;
 extern crate s3;
+extern crate tar;
+extern crate flate2;
 
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -43,6 +45,8 @@ use conduit::{Request, Method};
 use conduit_test::MockRequest;
 use diesel::prelude::*;
 use diesel::pg::upsert::*;
+use flate2::Compression;
+use flate2::write::GzEncoder;
 
 macro_rules! t {
     ($e:expr) => (
@@ -131,8 +135,8 @@ fn app() -> (record::Bomb, Arc<App>, conduit_middleware::MiddlewareBuilder) {
         bucket: s3::Bucket::new(
             String::from("alexcrichton-test"),
             None,
-            String::new(),
-            String::new(),
+            std::env::var("S3_ACCESS_KEY").unwrap_or(String::new()),
+            std::env::var("S3_SECRET_KEY").unwrap_or(String::new()),
             &api_protocol,
         ),
         proxy: Some(proxy),
@@ -631,6 +635,7 @@ fn new_req_body(
 ) -> Vec<u8> {
     let kws = kws.into_iter().map(u::Keyword).collect();
     let cats = cats.into_iter().map(u::Category).collect();
+
     new_crate_to_body(
         &u::NewCrate {
             name: u::CrateName(krate.name),
@@ -653,7 +658,19 @@ fn new_req_body(
     )
 }
 
-fn new_crate_to_body(new_crate: &u::NewCrate, krate: &[u8]) -> Vec<u8> {
+fn new_crate_to_body(new_crate: &u::NewCrate, files: &[(&str, &[u8])]) -> Vec<u8> {
+    let mut tarball = Vec::new();
+    {
+        let mut ar = tar::Builder::new(GzEncoder::new(&mut tarball, Compression::Default));
+        for &(name, data) in files {
+            let mut header = tar::Header::new_gnu();
+            t!(header.set_path(name));
+            header.set_size(data.len() as u64);
+            header.set_cksum();
+            t!(ar.append(&header, &data[..]));
+        }
+        t!(ar.finish());
+    }
     let json = serde_json::to_string(&new_crate).unwrap();
     let mut body = Vec::new();
     body.extend(
@@ -668,12 +685,12 @@ fn new_crate_to_body(new_crate: &u::NewCrate, krate: &[u8]) -> Vec<u8> {
     body.extend(json.as_bytes().iter().cloned());
     body.extend(
         &[
-            (krate.len() >> 0) as u8,
-            (krate.len() >> 8) as u8,
-            (krate.len() >> 16) as u8,
-            (krate.len() >> 24) as u8,
+            (tarball.len() >> 0) as u8,
+            (tarball.len() >> 8) as u8,
+            (tarball.len() >> 16) as u8,
+            (tarball.len() >> 24) as u8,
         ],
     );
-    body.extend(krate);
+    body.extend(tarball);
     body
 }

src/tests/all.rs

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_appveyor_extra_keys

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_circle_ci_required_keys

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_clear_badges

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_codecov_required_keys

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_coveralls_required_keys

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_gitlab_required_keys

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_isitmaintained_issue_resolution_required_keys

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_isitmaintained_open_issues_required_keys

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_maintenance_invalid_values

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_maintenance_required_keys

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_replace_badge

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_travis_ci_required_keys

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_unknown_badge

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_add_appveyor

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_add_circle_ci

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_add_codecov

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_add_coveralls

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_add_gitlab

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_add_isitmaintained_issue_resolution

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_add_isitmaintained_open_issues

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_add_maintenance

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_add_travis_ci

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_attributes

@@ -0,0 +1 @@
+[]

src/tests/http-data/badge_update_no_badges

@@ -0,0 +1 @@
+[]

src/tests/http-data/category_category_slugs_returns_all_slugs_in_alphabetical_order

@@ -0,0 +1 @@
+[]

src/tests/http-data/category_index

@@ -0,0 +1 @@
+[]

src/tests/http-data/category_show

@@ -0,0 +1 @@
+[]

src/tests/http-data/category_update_crate

@@ -1 +1 @@
-28f9838158f8dcbdbc01089634d7a14f495fb8f6
+69551e7bd735be9c59d3702a7885669979f0cdbc

src/tests/http-data/gh-crates-tester-1

@@ -1 +1 @@
-aa4e8e9caaaff1f7af211c2bc7d96b4202992e13
+7534f8b996e3a3f800f0a324f619adba12a74532

src/tests/http-data/gh-crates-tester-2

@@ -0,0 +1 @@
+[]

src/tests/http-data/keyword_index

@@ -0,0 +1 @@
+[]

src/tests/http-data/keyword_show

@@ -0,0 +1 @@
+[]

src/tests/http-data/keyword_update_crate

@@ -0,0 +1 @@
+[]

src/tests/http-data/keyword_uppercase

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_author_license_and_description_required

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_bad_keywords

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_block_blacklisted_documentation_url

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_dependencies

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_diesel_not_found_results_in_404

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_download

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_download_bad

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_exact_match_first_on_queries

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_exact_match_on_queries_with_sort

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_following

@@ -1,21 +1 @@
-===REQUEST 343
-PUT http://alexcrichton-test.s3.amazonaws.com/crates/foobadger/foobadger-1.0.0.crate HTTP/1.1
-Accept: */*
-Proxy-Connection: Keep-Alive
-Authorization: AWS AKIAJF3GEK7N44BACDZA:GDxGb6r3SIqo9wXuzHrgMNWekwk=
-Content-Length: 0
-Host: alexcrichton-test.s3.amazonaws.com
-Content-Type: application/x-tar
-Date: Sun, 28 Jun 2015 14:07:17 -0700
-
-
-===RESPONSE 258
-HTTP/1.1 200
-x-amz-request-id: CB0E925D8E3AB3E8
-x-amz-id-2: SiaMwszM1p2TzXlLauvZ6kRKcUCg7HoyBW29vts42w9ArrLwkJWl8vuvPuGFkpM6XGH+YXN852g=
-date: Sun, 28 Jun 2015 21:07:51 GMT
-etag: "d41d8cd98f00b204e9800998ecf8427e"
-content-length: 0
-server: AmazonS3
-
-
+[{"request":{"uri":"http://alexcrichton-test.s3.amazonaws.com/crates/foobadger/foobadger-1.0.0.crate","method":"PUT","headers":[["Content-Length","35"],["Date","Thu, 14 Sep 2017 09:44:04 -0700"],["Host","alexcrichton-test.s3.amazonaws.com"],["Proxy-Connection","Keep-Alive"],["Accept","*/*"],["Authorization","AWS AKIAICL5IWUZYWWKA7JA:SJNsc3Rh+zV7dPHUQGr8Z4bO4ro="],["Content-Type","application/x-tar"]],"body":[31,139,8,0,0,0,0,0,0,3,237,192,1,1,0,0,0,130,32,255,175,110,72,80,192,171,1,46,175,181,239,0,4,0,0]},"response":{"status":200,"headers":[["x-amz-request-id","259B82C2DC8C8C75"],["Date","Thu, 14 Sep 2017 16:44:06 GMT"],["ETag","\"fc9f988c1ba7ccea220952fd61213979\""],["Server","AmazonS3"],["x-amz-id-2","nej/HWPKXhTiYTCv+j1kSPOsoEHSJIeTaHnc2kbeLGflGKle+nGm5WqOUIKLKaRuF/IJFfG9bCI="],["Content-Length","0"]],"body":[]}}]

src/tests/http-data/krate_good_badges

@@ -1,21 +1 @@
-===REQUEST 349
-PUT http://alexcrichton-test.s3.amazonaws.com/crates/foo_good_cat/foo_good_cat-1.0.0.crate HTTP/1.1
-Accept: */*
-Proxy-Connection: Keep-Alive
-Authorization: AWS AKIAJF3GEK7N44BACDZA:GDxGb6r3SIqo9wXuzHrgMNWekwk=
-Content-Length: 0
-Host: alexcrichton-test.s3.amazonaws.com
-Content-Type: application/x-tar
-Date: Sun, 28 Jun 2015 14:07:17 -0700
-
-
-===RESPONSE 258
-HTTP/1.1 200
-x-amz-request-id: CB0E925D8E3AB3E8
-x-amz-id-2: SiaMwszM1p2TzXlLauvZ6kRKcUCg7HoyBW29vts42w9ArrLwkJWl8vuvPuGFkpM6XGH+YXN852g=
-date: Sun, 28 Jun 2015 21:07:51 GMT
-etag: "d41d8cd98f00b204e9800998ecf8427e"
-content-length: 0
-server: AmazonS3
-
-
+[{"request":{"uri":"http://alexcrichton-test.s3.amazonaws.com/crates/foo_good_cat/foo_good_cat-1.0.0.crate","method":"PUT","headers":[["Authorization","AWS AKIAICL5IWUZYWWKA7JA:3YdZWaxXYkwNl7cP7iSgeBmnFIA="],["Date","Thu, 14 Sep 2017 09:44:04 -0700"],["Content-Length","35"],["Content-Type","application/x-tar"],["Proxy-Connection","Keep-Alive"],["Accept","*/*"],["Host","alexcrichton-test.s3.amazonaws.com"]],"body":[31,139,8,0,0,0,0,0,0,3,237,192,1,1,0,0,0,130,32,255,175,110,72,80,192,171,1,46,175,181,239,0,4,0,0]},"response":{"status":200,"headers":[["x-amz-id-2","6doapP6obvc16pdSxMsxgRX+TUB+55dkNfsyw3ludfUfBOd2xhf2sL3aI2dKsnf5YCyl0QkEnXM="],["x-amz-request-id","D4BB18987F1E192F"],["Date","Thu, 14 Sep 2017 16:44:06 GMT"],["Server","AmazonS3"],["ETag","\"fc9f988c1ba7ccea220952fd61213979\""],["Content-Length","0"]],"body":[]}}]

src/tests/http-data/krate_good_categories

@@ -1,21 +1 @@
-===REQUEST 359
-PUT http://alexcrichton-test.s3.amazonaws.com/crates/foo_ignored_badge/foo_ignored_badge-1.0.0.crate HTTP/1.1
-Accept: */*
-Proxy-Connection: Keep-Alive
-Authorization: AWS AKIAJF3GEK7N44BACDZA:GDxGb6r3SIqo9wXuzHrgMNWekwk=
-Content-Length: 0
-Host: alexcrichton-test.s3.amazonaws.com
-Content-Type: application/x-tar
-Date: Sun, 28 Jun 2015 14:07:17 -0700
-
-
-===RESPONSE 258
-HTTP/1.1 200
-x-amz-request-id: CB0E925D8E3AB3E8
-x-amz-id-2: SiaMwszM1p2TzXlLauvZ6kRKcUCg7HoyBW29vts42w9ArrLwkJWl8vuvPuGFkpM6XGH+YXN852g=
-date: Sun, 28 Jun 2015 21:07:51 GMT
-etag: "d41d8cd98f00b204e9800998ecf8427e"
-content-length: 0
-server: AmazonS3
-
-
+[{"request":{"uri":"http://alexcrichton-test.s3.amazonaws.com/crates/foo_ignored_badge/foo_ignored_badge-1.0.0.crate","method":"PUT","headers":[["Date","Thu, 14 Sep 2017 09:44:04 -0700"],["Host","alexcrichton-test.s3.amazonaws.com"],["Authorization","AWS AKIAICL5IWUZYWWKA7JA:PkIayPRQSzIV9dhyoyKUg4bDSzs="],["Proxy-Connection","Keep-Alive"],["Accept","*/*"],["Content-Type","application/x-tar"],["Content-Length","35"]],"body":[31,139,8,0,0,0,0,0,0,3,237,192,1,1,0,0,0,130,32,255,175,110,72,80,192,171,1,46,175,181,239,0,4,0,0]},"response":{"status":200,"headers":[["Content-Length","0"],["ETag","\"fc9f988c1ba7ccea220952fd61213979\""],["x-amz-id-2","lbe1X5M1zPPgDYNv9DDBKekuO0e5+gK+XthTu9TMtODupHpjj5eb9GqeOQq0w4L6gqPxQBJWJqU="],["Server","AmazonS3"],["Date","Thu, 14 Sep 2017 16:44:06 GMT"],["x-amz-request-id","37C5421A868669E2"]],"body":[]}}]

src/tests/http-data/krate_ignored_badges

@@ -1,21 +1 @@
-===REQUEST 355
-PUT http://alexcrichton-test.s3.amazonaws.com/crates/foo_ignored_cat/foo_ignored_cat-1.0.0.crate HTTP/1.1
-Accept: */*
-Proxy-Connection: Keep-Alive
-Authorization: AWS AKIAJF3GEK7N44BACDZA:GDxGb6r3SIqo9wXuzHrgMNWekwk=
-Content-Length: 0
-Host: alexcrichton-test.s3.amazonaws.com
-Content-Type: application/x-tar
-Date: Sun, 28 Jun 2015 14:07:17 -0700
-
-
-===RESPONSE 258
-HTTP/1.1 200
-x-amz-request-id: CB0E925D8E3AB3E8
-x-amz-id-2: SiaMwszM1p2TzXlLauvZ6kRKcUCg7HoyBW29vts42w9ArrLwkJWl8vuvPuGFkpM6XGH+YXN852g=
-date: Sun, 28 Jun 2015 21:07:51 GMT
-etag: "d41d8cd98f00b204e9800998ecf8427e"
-content-length: 0
-server: AmazonS3
-
-
+[{"request":{"uri":"http://alexcrichton-test.s3.amazonaws.com/crates/foo_ignored_cat/foo_ignored_cat-1.0.0.crate","method":"PUT","headers":[["Content-Type","application/x-tar"],["Proxy-Connection","Keep-Alive"],["Accept","*/*"],["Content-Length","35"],["Authorization","AWS AKIAICL5IWUZYWWKA7JA:zN+ouGBQMXYwxnltlU50HqLR4D0="],["Host","alexcrichton-test.s3.amazonaws.com"],["Date","Thu, 14 Sep 2017 09:44:04 -0700"]],"body":[31,139,8,0,0,0,0,0,0,3,237,192,1,1,0,0,0,130,32,255,175,110,72,80,192,171,1,46,175,181,239,0,4,0,0]},"response":{"status":200,"headers":[["ETag","\"fc9f988c1ba7ccea220952fd61213979\""],["x-amz-id-2","MV8ehajFmRoufqKP0fwR1c8SRJWyQbdRYy97wJybWRu7Euo8A5kkNfJ6epnDfXJy3vkjxetK3tE="],["x-amz-request-id","8356C99E90EA73EB"],["Date","Thu, 14 Sep 2017 16:44:06 GMT"],["Content-Length","0"],["Server","AmazonS3"]],"body":[]}}]

src/tests/http-data/krate_ignored_categories

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_index

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_index_queries

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_new_bad_names

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_new_crate_similar_name

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_new_crate_similar_name_hyphen

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_new_crate_similar_name_underscore

@@ -1,21 +1 @@
-===REQUEST 339
-PUT http://alexcrichton-test.s3.amazonaws.com/crates/foo_new/foo_new-1.0.0.crate HTTP/1.1
-Accept: */*
-Proxy-Connection: Keep-Alive
-Authorization: AWS AKIAJF3GEK7N44BACDZA:GDxGb6r3SIqo9wXuzHrgMNWekwk=
-Content-Length: 0
-Host: alexcrichton-test.s3.amazonaws.com
-Content-Type: application/x-tar
-Date: Sun, 28 Jun 2015 14:07:17 -0700
-
-
-===RESPONSE 258
-HTTP/1.1 200
-x-amz-request-id: CB0E925D8E3AB3E8
-x-amz-id-2: SiaMwszM1p2TzXlLauvZ6kRKcUCg7HoyBW29vts42w9ArrLwkJWl8vuvPuGFkpM6XGH+YXN852g=
-date: Sun, 28 Jun 2015 21:07:51 GMT
-etag: "d41d8cd98f00b204e9800998ecf8427e"
-content-length: 0
-server: AmazonS3
-
-
+[{"request":{"uri":"http://alexcrichton-test.s3.amazonaws.com/crates/foo_new/foo_new-1.0.0.crate","method":"PUT","headers":[["Proxy-Connection","Keep-Alive"],["Accept","*/*"],["Content-Length","35"],["Host","alexcrichton-test.s3.amazonaws.com"],["Content-Type","application/x-tar"],["Date","Thu, 14 Sep 2017 09:44:05 -0700"],["Authorization","AWS AKIAICL5IWUZYWWKA7JA:wqYX81UuSZQN8fgc5iP80iGtuhQ="]],"body":[31,139,8,0,0,0,0,0,0,3,237,192,1,1,0,0,0,130,32,255,175,110,72,80,192,171,1,46,175,181,239,0,4,0,0]},"response":{"status":200,"headers":[["ETag","\"fc9f988c1ba7ccea220952fd61213979\""],["x-amz-request-id","43421DA20D1F6F8D"],["Date","Thu, 14 Sep 2017 16:44:06 GMT"],["Server","AmazonS3"],["Content-Length","0"],["x-amz-id-2","xVVFL9W1Qmv14Zgquz4gGVfNvTz+KF4TnWN2PaQXTnvq6HX2qFeQN1nCF45/gwEBrDTqNOZApFs="]],"body":[]}}]

src/tests/http-data/krate_new_krate

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_new_krate_bad_name

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_new_krate_dependency_missing

@@ -0,0 +1 @@
+[]

src/tests/http-data/krate_new_krate_duplicate_version

@@ -1,21 +1 @@
-===REQUEST 331
-PUT http://alexcrichton-test.s3.amazonaws.com/crates/fgt/fgt-1.0.0.crate HTTP/1.1
-Accept: */*
-Proxy-Connection: Keep-Alive
-Host: alexcrichton-test.s3.amazonaws.com
-Content-Length: 0
-Authorization: AWS AKIAJF3GEK7N44BACDZA:GDxGb6r3SIqo9wXuzHrgMNWekwk=
-Date: Sun, 28 Jun 2015 14:07:17 -0700
-Content-Type: application/x-tar
-
-
-===RESPONSE 258
-HTTP/1.1 200
-server: AmazonS3
-x-amz-id-2: jk22ONmcJ2OxrC/gvWL6Zs9gKDKzq3wvi2Cm0vapXjilMLb4UKmXk/PQP54S2Urq638cavjvgXA=
-x-amz-request-id: 6EAB09F4446CB1B0
-etag: "d41d8cd98f00b204e9800998ecf8427e"
-date: Sun, 28 Jun 2015 21:07:51 GMT
-content-length: 0
-
-
+[{"request":{"uri":"http://alexcrichton-test.s3.amazonaws.com/crates/fgt/fgt-1.0.0.crate","method":"PUT","headers":[["Proxy-Connection","Keep-Alive"],["Date","Thu, 14 Sep 2017 09:44:05 -0700"],["Authorization","AWS AKIAICL5IWUZYWWKA7JA:Hpi3LGRB4W3nXoPoX+A78xTzmIE="],["Accept","*/*"],["Host","alexcrichton-test.s3.amazonaws.com"],["Content-Type","application/x-tar"],["Content-Length","35"]],"body":[31,139,8,0,0,0,0,0,0,3,237,192,1,1,0,0,0,130,32,255,175,110,72,80,192,171,1,46,175,181,239,0,4,0,0]},"response":{"status":200,"headers":[["Content-Length","0"],["Server","AmazonS3"],["x-amz-request-id","5D8919DDF187F850"],["Date","Thu, 14 Sep 2017 16:44:06 GMT"],["x-amz-id-2","7N6yjY/oPkOJcDqcf5YSeABzp+hHEiwC67AXfbNqQNQAp0wRwU1c8KtN9Vh7vRshnmD/Hdjs6KY="],["ETag","\"fc9f988c1ba7ccea220952fd61213979\""]],"body":[]}}]