Skip to content

Commit 1e6881b

Browse files
Turbo87Turbo87
committed
Create crates.xml RSS feed
This creates an RSS feed published at https://static.crates.io/rss/crates.xml. The feed is synced with the database in a background job after every successful publish of a new crate. It includes the latest 50 newly created crates with their name, crate description, URL and publish date.
1 parent 27d505b commit 1e6881b

File tree

14 files changed

+276
-0
lines changed

14 files changed

+276
-0
lines changed

‎src/admin/enqueue_job.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ pub enum Command {
3939
force: bool,
4040
},
4141
SendTokenExpiryNotifications,
42+
SyncCratesFeed,
4243
SyncUpdatesFeed,
4344
}
4445

@@ -126,6 +127,9 @@ pub fn run(command: Command) -> Result<()> {
126127
Command::SendTokenExpiryNotifications => {
127128
jobs::SendTokenExpiryNotifications.enqueue(conn)?;
128129
}
130+
Command::SyncCratesFeed => {
131+
jobs::rss::SyncCratesFeed.enqueue(conn)?;
132+
}
129133
Command::SyncUpdatesFeed => {
130134
jobs::rss::SyncUpdatesFeed.enqueue(conn)?;
131135
}

‎src/controllers/krate/publish.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,12 @@ pub async fn publish(app: AppState, req: BytesRequest) -> AppResult<Json<GoodCra
450450
error!("Failed to enqueue `rss::SyncUpdatesFeed` job: {error}");
451451
}
452452

453+
if existing_crate.is_none() {
454+
if let Err(error) = jobs::rss::SyncCratesFeed.enqueue(conn) {
455+
error!("Failed to enqueue `rss::SyncCratesFeed` job: {error}");
456+
}
457+
}
458+
453459
// The `other` field on `PublishWarnings` was introduced to handle a temporary warning
454460
// that is no longer needed. As such, crates.io currently does not return any `other`
455461
// warnings at this time, but if we need to, the field is available.

‎src/storage.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,12 +376,14 @@ fn apply_cdn_prefix(cdn_prefix: &Option<String>, path: &Path) -> String {
376376

377377
#[derive(Debug)]
378378
pub enum FeedId {
379+
Crates,
379380
Updates,
380381
}
381382

382383
impl From<&FeedId> for Path {
383384
fn from(feed_id: &FeedId) -> Path {
384385
match feed_id {
386+
FeedId::Crates => "rss/crates.xml".into(),
385387
FeedId::Updates => "rss/updates.xml".into(),
386388
}
387389
}

‎src/tests/krate/publish/basics.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ async fn new_krate() {
2424
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
2525
crates/foo_new/foo_new-1.0.0.crate
2626
index/fo/o_/foo_new
27+
rss/crates.xml
2728
rss/updates.xml
2829
"###);
2930

@@ -51,6 +52,7 @@ async fn new_krate_with_token() {
5152
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
5253
crates/foo_new/foo_new-1.0.0.crate
5354
index/fo/o_/foo_new
55+
rss/crates.xml
5456
rss/updates.xml
5557
"###);
5658
}
@@ -70,6 +72,7 @@ async fn new_krate_weird_version() {
7072
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
7173
crates/foo_weird/foo_weird-0.0.0-pre.crate
7274
index/fo/o_/foo_weird
75+
rss/crates.xml
7376
rss/updates.xml
7477
"###);
7578
}
@@ -97,6 +100,7 @@ async fn new_krate_twice() {
97100
crates/foo_twice/foo_twice-0.99.0.crate
98101
crates/foo_twice/foo_twice-2.0.0.crate
99102
index/fo/o_/foo_twice
103+
rss/crates.xml
100104
rss/updates.xml
101105
"###);
102106
}

‎src/tests/krate/publish/git.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ async fn new_krate_git_upload_with_conflicts() {
1414
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
1515
crates/foo_conflicts/foo_conflicts-1.0.0.crate
1616
index/fo/o_/foo_conflicts
17+
rss/crates.xml
1718
rss/updates.xml
1819
"###);
1920
}

‎src/tests/krate/publish/max_size.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ async fn tarball_between_default_axum_limit_and_max_upload_size() {
5252
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
5353
crates/foo/foo-1.1.0.crate
5454
index/3/f/foo
55+
rss/crates.xml
5556
rss/updates.xml
5657
"###);
5758
}

‎src/tests/krate/publish/rate_limit.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ async fn publish_new_crate_ratelimit_expires() {
6969
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
7070
crates/rate_limited/rate_limited-1.0.0.crate
7171
index/ra/te/rate_limited
72+
rss/crates.xml
7273
rss/updates.xml
7374
"###);
7475

@@ -106,6 +107,7 @@ async fn publish_new_crate_override_loosens_ratelimit() {
106107
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
107108
crates/rate_limited1/rate_limited1-1.0.0.crate
108109
index/ra/te/rate_limited1
110+
rss/crates.xml
109111
rss/updates.xml
110112
"###);
111113

@@ -120,6 +122,7 @@ async fn publish_new_crate_override_loosens_ratelimit() {
120122
crates/rate_limited2/rate_limited2-1.0.0.crate
121123
index/ra/te/rate_limited1
122124
index/ra/te/rate_limited2
125+
rss/crates.xml
123126
rss/updates.xml
124127
"###);
125128

@@ -137,6 +140,7 @@ async fn publish_new_crate_override_loosens_ratelimit() {
137140
crates/rate_limited2/rate_limited2-1.0.0.crate
138141
index/ra/te/rate_limited1
139142
index/ra/te/rate_limited2
143+
rss/crates.xml
140144
rss/updates.xml
141145
"###);
142146

@@ -175,6 +179,7 @@ async fn publish_new_crate_expired_override_ignored() {
175179
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
176180
crates/rate_limited1/rate_limited1-1.0.0.crate
177181
index/ra/te/rate_limited1
182+
rss/crates.xml
178183
rss/updates.xml
179184
"###);
180185

@@ -190,6 +195,7 @@ async fn publish_new_crate_expired_override_ignored() {
190195
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
191196
crates/rate_limited1/rate_limited1-1.0.0.crate
192197
index/ra/te/rate_limited1
198+
rss/crates.xml
193199
rss/updates.xml
194200
"###);
195201

@@ -226,6 +232,7 @@ async fn publish_existing_crate_rate_limited() {
226232
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
227233
crates/rate_limited1/rate_limited1-1.0.0.crate
228234
index/ra/te/rate_limited1
235+
rss/crates.xml
229236
rss/updates.xml
230237
"###);
231238

@@ -239,6 +246,7 @@ async fn publish_existing_crate_rate_limited() {
239246
crates/rate_limited1/rate_limited1-1.0.0.crate
240247
crates/rate_limited1/rate_limited1-1.0.1.crate
241248
index/ra/te/rate_limited1
249+
rss/crates.xml
242250
rss/updates.xml
243251
"###);
244252

@@ -256,6 +264,7 @@ async fn publish_existing_crate_rate_limited() {
256264
crates/rate_limited1/rate_limited1-1.0.0.crate
257265
crates/rate_limited1/rate_limited1-1.0.1.crate
258266
index/ra/te/rate_limited1
267+
rss/crates.xml
259268
rss/updates.xml
260269
"###);
261270

@@ -272,6 +281,7 @@ async fn publish_existing_crate_rate_limited() {
272281
crates/rate_limited1/rate_limited1-1.0.1.crate
273282
crates/rate_limited1/rate_limited1-1.0.2.crate
274283
index/ra/te/rate_limited1
284+
rss/crates.xml
275285
rss/updates.xml
276286
"###);
277287
}

‎src/tests/krate/publish/readme.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ async fn new_krate_with_readme() {
1919
crates/foo_readme/foo_readme-1.0.0.crate
2020
index/fo/o_/foo_readme
2121
readmes/foo_readme/foo_readme-1.0.0.html
22+
rss/crates.xml
2223
rss/updates.xml
2324
"###);
2425
}
@@ -38,6 +39,7 @@ async fn new_krate_with_empty_readme() {
3839
assert_snapshot!(app.stored_files().await.join("\n"), @r###"
3940
crates/foo_readme/foo_readme-1.0.0.crate
4041
index/fo/o_/foo_readme
42+
rss/crates.xml
4143
rss/updates.xml
4244
"###);
4345
}
@@ -58,6 +60,7 @@ async fn new_krate_with_readme_and_plus_version() {
5860
crates/foo_readme/foo_readme-1.0.0+foo.crate
5961
index/fo/o_/foo_readme
6062
readmes/foo_readme/foo_readme-1.0.0+foo.html
63+
rss/crates.xml
6164
rss/updates.xml
6265
"###);
6366
}

‎src/tests/worker/rss/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
mod sync_crates_feed;
12
mod sync_updates_feed;

‎src/tests/worker/rss/snapshots/all__worker__rss__sync_crates_feed__sync_crates_feed-2.snap

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
---
2+
source: src/tests/worker/rss/sync_crates_feed.rs
3+
expression: content
4+
---
5+
<?xml version="1.0" encoding="utf-8"?>
6+
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:crates="https://crates.io/">
7+
<channel>
8+
<title>crates.io: newest crates</title>
9+
<link>https://crates.io/</link>
10+
<description>Newest crates registered on the crates.io package registry</description>
11+
<language>en</language>
12+
<atom:link href="https://static.crates.io/rss/crates.xml" rel="self" type="application/rss+xml"/>
13+
<item>
14+
<title>New crate created: quux</title>
15+
<link>https://crates.io/crates/quux</link>
16+
<guid>https://crates.io/crates/quux</guid>
17+
<pubDate>Fri, 21 Jun 2024 17:03:45 +0000</pubDate>
18+
<crates:name>quux</crates:name>
19+
</item>
20+
<item>
21+
<title>New crate created: baz</title>
22+
<link>https://crates.io/crates/baz</link>
23+
<description><![CDATA[does it handle XML? &lt;item&gt;]]></description>
24+
<guid>https://crates.io/crates/baz</guid>
25+
<pubDate>Fri, 21 Jun 2024 17:01:33 +0000</pubDate>
26+
<crates:name>baz</crates:name>
27+
</item>
28+
<item>
29+
<title>New crate created: bar</title>
30+
<link>https://crates.io/crates/bar</link>
31+
<guid>https://crates.io/crates/bar</guid>
32+
<pubDate>Thu, 20 Jun 2024 12:45:12 +0000</pubDate>
33+
<crates:name>bar</crates:name>
34+
</item>
35+
<item>
36+
<title>New crate created: foo</title>
37+
<link>https://crates.io/crates/foo</link>
38+
<description><![CDATA[something something foo]]></description>
39+
<guid>https://crates.io/crates/foo</guid>
40+
<pubDate>Thu, 20 Jun 2024 10:13:54 +0000</pubDate>
41+
<crates:name>foo</crates:name>
42+
</item>
43+
</channel>
44+
</rss>

‎src/tests/worker/rss/sync_crates_feed.rs

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
use crate::util::TestApp;
2+
use chrono::DateTime;
3+
use crates_io::schema::crates;
4+
use crates_io::worker::jobs;
5+
use crates_io_worker::BackgroundJob;
6+
use diesel::prelude::*;
7+
use diesel::{PgConnection, RunQueryDsl};
8+
use insta::assert_snapshot;
9+
10+
#[tokio::test(flavor = "multi_thread")]
11+
async fn test_sync_crates_feed() {
12+
let (app, _) = TestApp::full().empty();
13+
14+
app.db(|conn| {
15+
create_crate(
16+
conn,
17+
"foo",
18+
Some("something something foo"),
19+
"2024-06-20T10:13:54Z",
20+
);
21+
create_crate(conn, "bar", None, "2024-06-20T12:45:12Z");
22+
create_crate(
23+
conn,
24+
"baz",
25+
Some("does it handle XML? <item>"),
26+
"2024-06-21T17:01:33Z",
27+
);
28+
create_crate(conn, "quux", None, "2024-06-21T17:03:45Z");
29+
30+
jobs::rss::SyncCratesFeed.enqueue(conn).unwrap();
31+
});
32+
33+
app.run_pending_background_jobs().await;
34+
35+
assert_snapshot!(app.stored_files().await.join("\n"), @"rss/crates.xml");
36+
37+
let store = app.as_inner().storage.as_inner();
38+
let result = store.get(&"rss/crates.xml".into()).await.unwrap();
39+
let bytes = result.bytes().await.unwrap();
40+
let content = String::from_utf8(bytes.to_vec()).unwrap();
41+
assert_snapshot!(content);
42+
}
43+
44+
fn create_crate(
45+
conn: &mut PgConnection,
46+
name: &str,
47+
description: Option<&str>,
48+
publish_time: &str,
49+
) {
50+
let publish_time = DateTime::parse_from_rfc3339(publish_time)
51+
.unwrap()
52+
.naive_utc();
53+
54+
diesel::insert_into(crates::table)
55+
.values((
56+
crates::name.eq(name),
57+
crates::description.eq(description),
58+
crates::created_at.eq(publish_time),
59+
crates::updated_at.eq(publish_time),
60+
))
61+
.execute(conn)
62+
.unwrap();
63+
}

‎src/worker/jobs/rss/mod.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
mod sync_crates_feed;
12
mod sync_updates_feed;
23

4+
pub use sync_crates_feed::SyncCratesFeed;
35
pub use sync_updates_feed::SyncUpdatesFeed;

0 commit comments

Comments
 (0)