Merge #1498

1498: Add infrastructure for pagerduty integration r=carols10cents a=sgrif

Right our pagerduty notifications come from heroku email alerts and a
pingdom integration. We want to start paging on more specific metrics.
So far we've discussed:

- When no crate is uploaded for N period of time (likely 4 hours)
- When a background job has failed N times (likely 5)
- When the background queue has N jobs (likely 5)

In order to do this we need to have some code in place that actually
starts an incident and pages whoever is on call. We'd also like to test
this code before we start relying on it, so I've added a binary we can
use to make sure everything is configured correctly.

Co-authored-by: Sean Griffin <[email protected]>
Co-authored-by: Carol (Nichols || Goulding) <[email protected]>

Author: 3 people

Cargo.lock

@@ -0,0 +1,70 @@
+extern crate cargo_registry;
+extern crate reqwest;
+
+use self::reqwest::{header, StatusCode as Status};
+use std::env;
+
+use self::cargo_registry::util::*;
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "snake_case", tag = "event_type")]
+pub enum Event {
+    Trigger {
+        incident_key: Option<String>,
+        description: String,
+    },
+    Acknowledge {
+        incident_key: String,
+        description: Option<String>,
+    },
+    Resolve {
+        incident_key: String,
+        description: Option<String>,
+    },
+}
+
+impl Event {
+    /// Sends the event to pagerduty.
+    ///
+    /// If the variant is `Trigger`, this will page whoever is on call
+    /// (potentially waking them up at 3 AM).
+    pub fn send(self) -> CargoResult<()> {
+        let api_token = env::var("PAGERDUTY_API_TOKEN")?;
+        let service_key = env::var("PAGERDUTY_INTEGRATION_KEY")?;
+
+        let mut response = reqwest::Client::new()
+            .post("https://events.pagerduty.com/generic/2010-04-15/create_event.json")
+            .header(header::ACCEPT, "application/vnd.pagerduty+json;version=2")
+            .header(header::AUTHORIZATION, format!("Token token={}", api_token))
+            .json(&FullEvent {
+                service_key,
+                event: self,
+            }).send()?;
+
+        match response.status() {
+            s if s.is_success() => Ok(()),
+            Status::BAD_REQUEST => {
+                let error = response.json::<InvalidEvent>()?;
+                Err(internal(&format_args!("pagerduty error: {:?}", error)))
+            }
+            Status::FORBIDDEN => Err(internal("rate limited by pagerduty")),
+            n => Err(internal(&format_args!(
+                "Got a non 200 response code from pagerduty: {} with {:?}",
+                n, response
+            ))),
+        }
+    }
+}
+
+#[derive(Serialize, Debug)]
+struct FullEvent {
+    service_key: String,
+    #[serde(flatten)]
+    event: Event,
+}
+
+#[derive(Deserialize, Debug)]
+struct InvalidEvent {
+    message: String,
+    errors: Vec<String>,
+}

src/bin/on_call/mod.rs

@@ -0,0 +1,39 @@
+//! Send a test event to pagerduty
+//!
+//! Usage:
+//!     cargo run --bin test-pagerduty event_type [description]
+//!
+//! Event type can be trigger, acknowledge, or resolve
+
+#![deny(warnings)]
+
+#[macro_use]
+extern crate serde_derive;
+
+mod on_call;
+
+use std::env::args;
+
+fn main() {
+    let args = args().collect::<Vec<_>>();
+
+    let event_type = &*args[1];
+    let description = args.get(2).cloned();
+
+    let event = match event_type {
+        "trigger" => on_call::Event::Trigger {
+            incident_key: Some("test".into()),
+            description: description.unwrap_or_else(|| "Test event".into()),
+        },
+        "acknowledge" => on_call::Event::Acknowledge {
+            incident_key: "test".into(),
+            description,
+        },
+        "resolve" => on_call::Event::Resolve {
+            incident_key: "test".into(),
+            description,
+        },
+        _ => panic!("Event type must be trigger, acknowledge, or resolve"),
+    };
+    event.send().unwrap()
+}