Merge remote-tracking branch 'upstream/master' into sg-test-pagerduty

Author: carols10cents

.travis.yml

@@ -47,16 +47,15 @@ matrix:
   allow_failures:
     - rust: nightly
   include:
-    - rust: nightly-2018-06-26
-      script:
-        - cargo install --force clippy --vers 0.0.210
-        - cargo clippy
     - rust: stable
       before_install:
         - nvm install 10
         - rustup component add rustfmt-preview
+        - rustup component add clippy-preview
       script:
         - cargo fmt -- --check
+        # TODO: Once clippy:: lint syntax is stable, remove everything after the -- below
+        - cargo clippy --all-targets -- -A renamed_and_removed_lints
         - cargo build
         - cargo test
         - npm install

Cargo.lock

@@ -28,12 +28,14 @@ rustdoc-args = [
 
 [dependencies]
 cargo-registry-s3 = { path = "src/s3", version = "0.2.0" }
+old_semver = { path = "src/old_semver", version = "0.1.0" }
 rand = "0.3"
 git2 = "0.6.4"
 flate2 = "1.0"
-semver = "0.5"
+semver = { version = "0.9", git = "https://github.com/steveklabnik/semver.git", features = ["diesel", "serde"] }
 url = "1.2.1"
 tar = "0.4.13"
+base64 = "0.9"
 
 openssl = "0.9.14"
 oauth2 = "0.3"
@@ -58,6 +60,7 @@ itertools = "0.6.0"
 scheduled-thread-pool = "0.2.0"
 derive_deref = "1.0.0"
 reqwest = "0.9.1"
+tempdir = "0.3.7"
 
 lettre = "0.8"
 lettre_email = "0.8"
@@ -77,6 +80,7 @@ conduit-test = "0.8"
 hyper = "0.12"
 hyper-tls = "0.2"
 futures = "0.1"
+lazy_static = "1.0"
 tokio-core = "0.1"
 tokio-service = "0.1"
 

Cargo.toml

@@ -30,8 +30,8 @@ export default Controller.extend({
     keywords: alias('crate.keywords'),
     categories: alias('crate.categories'),
     badges: alias('crate.badges'),
-    isOwner: computed('crate.owner_user', 'session.currentUser.login', function() {
-        return this.get('crate.owner_user').findBy('login', this.get('session.currentUser.login'));
+    isOwner: computed('crate.owner_user', 'session.currentUser.id', function() {
+        return this.get('crate.owner_user').findBy('id', this.get('session.currentUser.id'));
     }),
 
     sortedVersions: readOnly('crate.versions'),

app/controllers/crate/version.js

@@ -38,7 +38,7 @@
         @include align-items(center);
     }
     @media only screen and (max-width: 650px) {
-        .right { display: none; }
+        .right { @include justify-content(center); }
     }
 
     &.crate-index, &.crate-search {

app/styles/crate.scss

@@ -0,0 +1 @@
+DROP INDEX index_crates_name_ordering;

migrations/2018-10-17-203221_index_crates_name/down.sql

@@ -0,0 +1 @@
+CREATE INDEX index_crates_name_ordering ON crates (name);

migrations/2018-10-17-203221_index_crates_name/up.sql

@@ -8,7 +8,7 @@ use util::bad_request;
 
 use models::{Email, Follow, NewEmail, User, Version};
 use schema::{crates, emails, follows, users, versions};
-use views::{EncodablePrivateUser, EncodableVersion};
+use views::{EncodableMe, EncodableVersion};
 
 /// Handles the `GET /me` route.
 pub fn me(req: &mut dyn Request) -> CargoResult<Response> {
@@ -40,11 +40,7 @@ pub fn me(req: &mut dyn Request) -> CargoResult<Response> {
     let verification_sent = verified || verification_sent;
     let user = User { email, ..user };
 
-    #[derive(Serialize)]
-    struct R {
-        user: EncodablePrivateUser,
-    }
-    Ok(req.json(&R {
+    Ok(req.json(&EncodableMe {
         user: user.encodable_private(verified, verification_sent),
     }))
 }

src/controllers/user/me.rs

@@ -36,6 +36,7 @@ pub trait RequestTransaction {
     /// Return the lazily initialized postgres connection for this request.
     ///
     /// The connection will live for the lifetime of the request.
+    // FIXME: This description does not match the implementation below.
     fn db_conn(&self) -> CargoResult<DieselPooledConn>;
 }
 

src/db.rs

@@ -7,7 +7,7 @@
 #![deny(missing_debug_implementations, missing_copy_implementations)]
 #![deny(bare_trait_objects)]
 #![recursion_limit = "256"]
-#![allow(unknown_lints, proc_macro_derive_resolution_fallback)] // This can be removed after diesel-1.4
+#![allow(unknown_lints, proc_macro_derive_resolution_fallback)] // TODO: This can be removed after diesel-1.4
 
 extern crate ammonia;
 extern crate chrono;

src/lib.rs

@@ -27,13 +27,13 @@ impl AroundMiddleware for BlockIps {
 
 impl Handler for BlockIps {
     fn call(&self, req: &mut dyn Request) -> Result<Response, Box<dyn Error + Send>> {
-        let has_blacklisted_ip = req
+        let has_blocked_ip = req
             .headers()
             .find("X-Forwarded-For")
             .unwrap()
             .iter()
             .any(|v| v.split(", ").any(|ip| self.ips.iter().any(|x| x == ip)));
-        if has_blacklisted_ip {
+        if has_blocked_ip {
             let body = format!(
                 "We are unable to process your request at this time. \
                  Please open an issue at https://github.com/rust-lang/crates.io \

src/middleware/blacklist_ips.rs renamed to src/middleware/block_ips.rs

@@ -13,13 +13,14 @@ pub use self::security_headers::SecurityHeaders;
 pub use self::static_or_continue::StaticOrContinue;
 
 pub mod app;
-mod blacklist_ips;
+mod block_ips;
 pub mod current_user;
 mod debug;
 mod ember_index_rewrite;
 mod ensure_well_formed_500;
 mod head;
 mod log_request;
+mod require_user_agent;
 mod security_headers;
 mod static_or_continue;
 
@@ -81,10 +82,11 @@ pub fn build_middleware(app: Arc<App>, endpoints: R404) -> MiddlewareBuilder {
 
     m.around(Head::default());
 
-    if let Ok(ip_list) = env::var("BLACKLISTED_IPS") {
+    if let Ok(ip_list) = env::var("BLOCKED_IPS") {
         let ips = ip_list.split(',').map(String::from).collect();
-        m.around(blacklist_ips::BlockIps::new(ips));
+        m.around(block_ips::BlockIps::new(ips));
     }
+    m.around(require_user_agent::RequireUserAgent::default());
 
     if env != Env::Test {
         m.around(log_request::LogRequests::default());

src/middleware/mod.rs

@@ -0,0 +1,13 @@
+We require that all requests include a `User-Agent` header.  To allow us to determine the impact your bot has on our service, we ask that your user agent actually identify your bot, and not just report the HTTP client library you're using.  Including contact information will also reduce the chance that we will need to take action against your bot.
+
+Bad:
+  User-Agent: reqwest/0.9.1
+
+Better:
+  User-Agent: my_crawler
+
+Best:
+  User-Agent: my_crawler (my_crawler.com/info)
+  User-Agent: my_crawler (help@my_crawler.com)
+
+If you believe you've received this message in error, please email [email protected] and include the request id {}.

src/middleware/no_user_agent_message.txt

@@ -0,0 +1,41 @@
+//! Middleware that blocks requests with no user-agent header
+
+use super::prelude::*;
+
+use std::collections::HashMap;
+use std::io::Cursor;
+use util::request_header;
+
+// Can't derive debug because of Handler.
+#[allow(missing_debug_implementations)]
+#[derive(Default)]
+pub struct RequireUserAgent {
+    handler: Option<Box<dyn Handler>>,
+}
+
+impl AroundMiddleware for RequireUserAgent {
+    fn with_handler(&mut self, handler: Box<dyn Handler>) {
+        self.handler = Some(handler);
+    }
+}
+
+impl Handler for RequireUserAgent {
+    fn call(&self, req: &mut dyn Request) -> Result<Response, Box<dyn Error + Send>> {
+        let has_user_agent = request_header(req, "User-Agent") != "";
+        if !has_user_agent {
+            let body = format!(
+                include_str!("no_user_agent_message.txt"),
+                request_header(req, "X-Request-Id"),
+            );
+            let mut headers = HashMap::new();
+            headers.insert("Content-Length".to_string(), vec![body.len().to_string()]);
+            Ok(Response {
+                status: (403, "Forbidden"),
+                headers,
+                body: Box::new(Cursor::new(body.into_bytes())),
+            })
+        } else {
+            self.handler.as_ref().unwrap().call(req)
+        }
+    }
+}

src/middleware/require_user_agent.rs

@@ -1,7 +1,4 @@
-use diesel::deserialize::QueryableByName;
-use diesel::pg::Pg;
 use diesel::prelude::*;
-use diesel::row::NamedRow;
 use semver;
 
 use git;
@@ -11,7 +8,7 @@ use models::{Crate, Version};
 use schema::*;
 use views::{EncodableCrateDependency, EncodableDependency};
 
-#[derive(Identifiable, Associations, Debug)]
+#[derive(Identifiable, Associations, Debug, Queryable, QueryableByName)]
 #[belongs_to(Version)]
 #[belongs_to(Crate)]
 #[table_name = "dependencies"]
@@ -141,6 +138,7 @@ pub fn add_dependencies(
 }
 
 use diesel::deserialize::{self, FromSql};
+use diesel::pg::Pg;
 use diesel::sql_types::Integer;
 
 impl FromSql<Integer, Pg> for DependencyKind {
@@ -153,51 +151,3 @@ impl FromSql<Integer, Pg> for DependencyKind {
         }
     }
 }
-
-impl Queryable<dependencies::SqlType, Pg> for Dependency {
-    type Row = (
-        i32,
-        i32,
-        i32,
-        String,
-        bool,
-        bool,
-        Vec<String>,
-        Option<String>,
-        DependencyKind,
-    );
-
-    fn build(row: Self::Row) -> Self {
-        Dependency {
-            id: row.0,
-            version_id: row.1,
-            crate_id: row.2,
-            req: semver::VersionReq::parse(&row.3).unwrap(),
-            optional: row.4,
-            default_features: row.5,
-            features: row.6,
-            target: row.7,
-            kind: row.8,
-        }
-    }
-}
-
-impl QueryableByName<Pg> for Dependency {
-    fn build<R: NamedRow<Pg>>(row: &R) -> deserialize::Result<Self> {
-        use diesel::dsl::SqlTypeOf;
-        use schema::dependencies::*;
-
-        let req_str = row.get::<SqlTypeOf<req>, String>("req")?;
-        Ok(Dependency {
-            id: row.get::<SqlTypeOf<id>, _>("id")?,
-            version_id: row.get::<SqlTypeOf<version_id>, _>("version_id")?,
-            crate_id: row.get::<SqlTypeOf<crate_id>, _>("crate_id")?,
-            req: semver::VersionReq::parse(&req_str)?,
-            optional: row.get::<SqlTypeOf<optional>, _>("optional")?,
-            default_features: row.get::<SqlTypeOf<default_features>, _>("default_features")?,
-            features: row.get::<SqlTypeOf<features>, _>("features")?,
-            target: row.get::<SqlTypeOf<target>, _>("target")?,
-            kind: row.get::<SqlTypeOf<kind>, _>("kind")?,
-        })
-    }
-}

src/models/dependency.rs

@@ -19,9 +19,9 @@ use views::{EncodableCrate, EncodableCrateLinks};
 use models::helpers::with_count::*;
 use schema::*;
 
-/// Hosts in this blacklist are known to not be hosting documentation,
+/// Hosts in this list are known to not be hosting documentation,
 /// and are possibly of malicious intent e.g. ad tracking networks, etc.
-const DOCUMENTATION_BLACKLIST: [&str; 1] = ["rust-ci.org"];
+const DOCUMENTATION_BLOCKLIST: [&str; 1] = ["rust-ci.org"];
 
 #[derive(Debug, Insertable, Queryable, Identifiable, Associations, AsChangeset, Clone, Copy)]
 #[belongs_to(Crate)]
@@ -330,7 +330,7 @@ impl Crate {
         let keyword_ids = keywords.map(|kws| kws.iter().map(|kw| kw.keyword.clone()).collect());
         let category_ids = categories.map(|cats| cats.iter().map(|cat| cat.slug.clone()).collect());
         let badges = badges.map(|bs| bs.into_iter().map(|b| b.encodable()).collect());
-        let documentation = Crate::remove_blacklisted_documentation_urls(documentation);
+        let documentation = Crate::remove_blocked_documentation_urls(documentation);
 
         EncodableCrate {
             id: name.clone(),
@@ -360,8 +360,8 @@ impl Crate {
         }
     }
 
-    /// Return `None` if the documentation URL host matches a blacklisted host
-    fn remove_blacklisted_documentation_urls(url: Option<String>) -> Option<String> {
+    /// Return `None` if the documentation URL host matches a blocked host
+    fn remove_blocked_documentation_urls(url: Option<String>) -> Option<String> {
         // Handles if documentation URL is None
         let url = match url {
             Some(url) => url,
@@ -380,8 +380,8 @@ impl Crate {
             None => return None,
         };
 
-        // Match documentation URL host against blacklisted host array elements
-        if DOCUMENTATION_BLACKLIST.contains(&url_host) {
+        // Match documentation URL host against blocked host array elements
+        if DOCUMENTATION_BLOCKLIST.contains(&url_host) {
             None
         } else {
             Some(url)
@@ -520,32 +520,32 @@ mod tests {
     use models::Crate;
 
     #[test]
-    fn documentation_blacklist_no_url_provided() {
-        assert_eq!(Crate::remove_blacklisted_documentation_urls(None), None);
+    fn documentation_blocked_no_url_provided() {
+        assert_eq!(Crate::remove_blocked_documentation_urls(None), None);
     }
 
     #[test]
-    fn documentation_blacklist_invalid_url() {
+    fn documentation_blocked_invalid_url() {
         assert_eq!(
-            Crate::remove_blacklisted_documentation_urls(Some(String::from("not a url"))),
+            Crate::remove_blocked_documentation_urls(Some(String::from("not a url"))),
             None
         );
     }
 
     #[test]
-    fn documentation_blacklist_url_contains_partial_match() {
+    fn documentation_blocked_url_contains_partial_match() {
         assert_eq!(
-            Crate::remove_blacklisted_documentation_urls(Some(String::from(
+            Crate::remove_blocked_documentation_urls(Some(String::from(
                 "http://rust-ci.organists.com"
             )),),
             Some(String::from("http://rust-ci.organists.com"))
         );
     }
 
     #[test]
-    fn documentation_blacklist_blacklisted_url() {
+    fn documentation_blocked_url() {
         assert_eq!(
-            Crate::remove_blacklisted_documentation_urls(Some(String::from(
+            Crate::remove_blocked_documentation_urls(Some(String::from(
                 "http://rust-ci.org/crate/crate-0.1/doc/crate-0.1",
             ),),),
             None