Create updates.xml RSS feed

This creates an RSS feed published at https://static.crates.io/rss/updates.xml. The feed is synced with the database in a background job after every successful publish of a new version. It includes the latest 100 published versions with the crate name, version number, crate description, URL and publish date.

Author: Turbo87

Cargo.lock

@@ -98,8 +98,10 @@ p256 = "=0.13.2"
 parking_lot = "=0.12.3"
 paste = "=1.0.15"
 prometheus = { version = "=0.13.4", default-features = false }
+quick-xml = "=0.31.0"
 rand = "=0.8.5"
 reqwest = { version = "=0.12.5", features = ["blocking", "gzip", "json"] }
+rss = { version = "=2.0.8", default-features = false, features = ["atom"] }
 scheduled-thread-pool = "=0.2.7"
 secrecy = "=0.8.0"
 semver = { version = "=1.0.23", features = ["serde"] }

Cargo.toml

@@ -39,6 +39,7 @@ pub enum Command {
         force: bool,
     },
     SendTokenExpiryNotifications,
+    SyncUpdatesFeed,
 }
 
 pub fn run(command: Command) -> Result<()> {
@@ -125,6 +126,9 @@ pub fn run(command: Command) -> Result<()> {
         Command::SendTokenExpiryNotifications => {
             jobs::SendTokenExpiryNotifications.enqueue(conn)?;
         }
+        Command::SyncUpdatesFeed => {
+            jobs::rss::SyncUpdatesFeed.enqueue(conn)?;
+        }
     };
 
     Ok(())

src/admin/enqueue_job.rs

@@ -431,6 +431,10 @@ pub async fn publish(app: AppState, req: BytesRequest) -> AppResult<Json<GoodCra
                 CheckTyposquat::new(&krate.name).enqueue(conn)?;
             }
 
+            if let Err(error) = jobs::rss::SyncUpdatesFeed.enqueue(conn) {
+                error!("Failed to enqueue `rss::SyncUpdatesFeed` job: {error}");
+            }
+
             // The `other` field on `PublishWarnings` was introduced to handle a temporary warning
             // that is no longer needed. As such, crates.io currently does not return any `other`
             // warnings at this time, but if we need to, the field is available.

src/controllers/krate/publish.rs

@@ -7,9 +7,10 @@ use object_store::local::LocalFileSystem;
 use object_store::memory::InMemory;
 use object_store::path::Path;
 use object_store::prefix::PrefixStore;
-use object_store::{Attribute, Attributes, ClientOptions, ObjectStore, Result};
+use object_store::{Attribute, Attributes, ClientOptions, ObjectStore, PutPayload, Result};
 use secrecy::{ExposeSecret, SecretString};
 use std::fs;
+use std::io::Cursor;
 use std::path::PathBuf;
 use std::sync::Arc;
 use tokio::fs::File;
@@ -203,6 +204,11 @@ impl Storage {
         apply_cdn_prefix(&self.cdn_prefix, &readme_path(name, version)).replace('+', "%2B")
     }
 
+    /// Returns the URL of an uploaded RSS feed.
+    pub fn feed_url(&self, feed_id: &FeedId) -> String {
+        apply_cdn_prefix(&self.cdn_prefix, &feed_id.into()).replace('+', "%2B")
+    }
+
     #[instrument(skip(self))]
     pub async fn delete_all_crate_files(&self, name: &str) -> Result<()> {
         let prefix = format!("{PREFIX_CRATES}/{name}").into();
@@ -251,6 +257,25 @@ impl Storage {
         Ok(())
     }
 
+    #[instrument(skip(self, channel))]
+    pub async fn upload_feed(
+        &self,
+        feed_id: &FeedId,
+        channel: &rss::Channel,
+    ) -> anyhow::Result<()> {
+        let path = feed_id.into();
+
+        let mut buffer = Vec::new();
+        let mut cursor = Cursor::new(&mut buffer);
+        channel.pretty_write_to(&mut cursor, b' ', 4)?;
+        let payload = PutPayload::from_bytes(buffer.into());
+
+        let attributes = self.attrs([(Attribute::ContentType, "text/xml; charset=UTF-8")]);
+        let opts = attributes.into();
+        self.store.put_opts(&path, payload, opts).await?;
+        Ok(())
+    }
+
     #[instrument(skip(self, content))]
     pub async fn sync_index(&self, name: &str, content: Option<String>) -> Result<()> {
         let path = crates_io_index::Repository::relative_index_file_for_url(name).into();
@@ -349,6 +374,19 @@ fn apply_cdn_prefix(cdn_prefix: &Option<String>, path: &Path) -> String {
     }
 }
 
+#[derive(Debug)]
+pub enum FeedId {
+    Updates,
+}
+
+impl From<&FeedId> for Path {
+    fn from(feed_id: &FeedId) -> Path {
+        match feed_id {
+            FeedId::Updates => "rss/updates.xml".into(),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/storage.rs

@@ -24,6 +24,7 @@ async fn new_krate() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/foo_new/foo_new-1.0.0.crate
     index/fo/o_/foo_new
+    rss/updates.xml
     "###);
 
     app.db(|conn| {
@@ -50,6 +51,7 @@ async fn new_krate_with_token() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/foo_new/foo_new-1.0.0.crate
     index/fo/o_/foo_new
+    rss/updates.xml
     "###);
 }
 
@@ -68,6 +70,7 @@ async fn new_krate_weird_version() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/foo_weird/foo_weird-0.0.0-pre.crate
     index/fo/o_/foo_weird
+    rss/updates.xml
     "###);
 }
 
@@ -94,6 +97,7 @@ async fn new_krate_twice() {
     crates/foo_twice/foo_twice-0.99.0.crate
     crates/foo_twice/foo_twice-2.0.0.crate
     index/fo/o_/foo_twice
+    rss/updates.xml
     "###);
 }
 

src/tests/krate/publish/basics.rs

@@ -14,5 +14,6 @@ async fn new_krate_git_upload_with_conflicts() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/foo_conflicts/foo_conflicts-1.0.0.crate
     index/fo/o_/foo_conflicts
+    rss/updates.xml
     "###);
 }

src/tests/krate/publish/git.rs

@@ -52,6 +52,7 @@ async fn tarball_between_default_axum_limit_and_max_upload_size() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/foo/foo-1.1.0.crate
     index/3/f/foo
+    rss/updates.xml
     "###);
 }
 
@@ -146,5 +147,6 @@ async fn new_krate_too_big_but_whitelisted() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/foo_whitelist/foo_whitelist-1.1.0.crate
     index/fo/o_/foo_whitelist
+    rss/updates.xml
     "###);
 }

src/tests/krate/publish/max_size.rs

@@ -69,6 +69,7 @@ async fn publish_new_crate_ratelimit_expires() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/rate_limited/rate_limited-1.0.0.crate
     index/ra/te/rate_limited
+    rss/updates.xml
     "###);
 
     let json = anon.show_crate("rate_limited").await;
@@ -105,6 +106,7 @@ async fn publish_new_crate_override_loosens_ratelimit() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/rate_limited1/rate_limited1-1.0.0.crate
     index/ra/te/rate_limited1
+    rss/updates.xml
     "###);
 
     let json = anon.show_crate("rate_limited1").await;
@@ -118,6 +120,7 @@ async fn publish_new_crate_override_loosens_ratelimit() {
     crates/rate_limited2/rate_limited2-1.0.0.crate
     index/ra/te/rate_limited1
     index/ra/te/rate_limited2
+    rss/updates.xml
     "###);
 
     let json = anon.show_crate("rate_limited2").await;
@@ -134,6 +137,7 @@ async fn publish_new_crate_override_loosens_ratelimit() {
     crates/rate_limited2/rate_limited2-1.0.0.crate
     index/ra/te/rate_limited1
     index/ra/te/rate_limited2
+    rss/updates.xml
     "###);
 
     let response = anon.get::<()>("/api/v1/crates/rate_limited3").await;
@@ -171,6 +175,7 @@ async fn publish_new_crate_expired_override_ignored() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/rate_limited1/rate_limited1-1.0.0.crate
     index/ra/te/rate_limited1
+    rss/updates.xml
     "###);
 
     let json = anon.show_crate("rate_limited1").await;
@@ -185,6 +190,7 @@ async fn publish_new_crate_expired_override_ignored() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/rate_limited1/rate_limited1-1.0.0.crate
     index/ra/te/rate_limited1
+    rss/updates.xml
     "###);
 
     let response = anon.get::<()>("/api/v1/crates/rate_limited2").await;
@@ -220,6 +226,7 @@ async fn publish_existing_crate_rate_limited() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/rate_limited1/rate_limited1-1.0.0.crate
     index/ra/te/rate_limited1
+    rss/updates.xml
     "###);
 
     // Uploading the first update to the crate works
@@ -232,6 +239,7 @@ async fn publish_existing_crate_rate_limited() {
     crates/rate_limited1/rate_limited1-1.0.0.crate
     crates/rate_limited1/rate_limited1-1.0.1.crate
     index/ra/te/rate_limited1
+    rss/updates.xml
     "###);
 
     // Uploading the second update to the crate is rate limited
@@ -248,6 +256,7 @@ async fn publish_existing_crate_rate_limited() {
     crates/rate_limited1/rate_limited1-1.0.0.crate
     crates/rate_limited1/rate_limited1-1.0.1.crate
     index/ra/te/rate_limited1
+    rss/updates.xml
     "###);
 
     // Wait for the limit to be up
@@ -263,6 +272,7 @@ async fn publish_existing_crate_rate_limited() {
     crates/rate_limited1/rate_limited1-1.0.1.crate
     crates/rate_limited1/rate_limited1-1.0.2.crate
     index/ra/te/rate_limited1
+    rss/updates.xml
     "###);
 }
 

src/tests/krate/publish/rate_limit.rs

@@ -19,6 +19,7 @@ async fn new_krate_with_readme() {
     crates/foo_readme/foo_readme-1.0.0.crate
     index/fo/o_/foo_readme
     readmes/foo_readme/foo_readme-1.0.0.html
+    rss/updates.xml
     "###);
 }
 
@@ -37,6 +38,7 @@ async fn new_krate_with_empty_readme() {
     assert_snapshot!(app.stored_files().await.join("\n"), @r###"
     crates/foo_readme/foo_readme-1.0.0.crate
     index/fo/o_/foo_readme
+    rss/updates.xml
     "###);
 }
 
@@ -56,6 +58,7 @@ async fn new_krate_with_readme_and_plus_version() {
     crates/foo_readme/foo_readme-1.0.0+foo.crate
     index/fo/o_/foo_readme
     readmes/foo_readme/foo_readme-1.0.0+foo.html
+    rss/updates.xml
     "###);
 }
 

src/tests/krate/publish/readme.rs

@@ -1,2 +1,3 @@
 mod git;
+mod rss;
 mod sync_admins;

src/tests/worker/mod.rs

@@ -0,0 +1 @@
+mod sync_updates_feed;