Skip to content

Commit 469b6bd

Browse files
Turbo87Turbo87
committed
rss: Always include *all* new crates from the past 60 minutes
1 parent 9d50d0e commit 469b6bd

4 files changed

+226
-0
lines changed

‎src/worker/jobs/rss/snapshots/crates_io__worker__jobs__rss__sync_crates_feed__tests__load_version_updates-2.snap

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
---
2+
source: src/worker/jobs/rss/sync_crates_feed.rs
3+
expression: "new_crates.iter().map(|u| &u.name).collect::<Vec<_>>()"
4+
---
5+
[
6+
"crate-50",
7+
"crate-49",
8+
"crate-48",
9+
"crate-47",
10+
"crate-46",
11+
"crate-45",
12+
"crate-44",
13+
"crate-43",
14+
"crate-42",
15+
"crate-41",
16+
"crate-40",
17+
"crate-39",
18+
"crate-38",
19+
"crate-37",
20+
"crate-36",
21+
"crate-35",
22+
"crate-34",
23+
"crate-33",
24+
"crate-32",
25+
"crate-31",
26+
"crate-30",
27+
"crate-29",
28+
"crate-28",
29+
"crate-27",
30+
"crate-26",
31+
"crate-25",
32+
"crate-24",
33+
"crate-23",
34+
"crate-22",
35+
"crate-21",
36+
"crate-20",
37+
"crate-19",
38+
"crate-18",
39+
"crate-17",
40+
"crate-16",
41+
"crate-15",
42+
"crate-14",
43+
"crate-13",
44+
"crate-12",
45+
"crate-11",
46+
"crate-10",
47+
"crate-9",
48+
"crate-8",
49+
"crate-7",
50+
"crate-6",
51+
"crate-5",
52+
"crate-4",
53+
"crate-3",
54+
"crate-2",
55+
"crate-1",
56+
]

‎src/worker/jobs/rss/snapshots/crates_io__worker__jobs__rss__sync_crates_feed__tests__load_version_updates-3.snap

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
---
2+
source: src/worker/jobs/rss/sync_crates_feed.rs
3+
expression: "new_crates.iter().map(|u| &u.name).collect::<Vec<_>>()"
4+
---
5+
[
6+
"other-crate-60",
7+
"other-crate-59",
8+
"other-crate-58",
9+
"other-crate-57",
10+
"other-crate-56",
11+
"other-crate-55",
12+
"other-crate-54",
13+
"other-crate-53",
14+
"other-crate-52",
15+
"other-crate-51",
16+
"other-crate-50",
17+
"other-crate-49",
18+
"other-crate-48",
19+
"other-crate-47",
20+
"other-crate-46",
21+
"other-crate-45",
22+
"other-crate-44",
23+
"other-crate-43",
24+
"other-crate-42",
25+
"other-crate-41",
26+
"other-crate-40",
27+
"other-crate-39",
28+
"other-crate-38",
29+
"other-crate-37",
30+
"other-crate-36",
31+
"other-crate-35",
32+
"other-crate-34",
33+
"other-crate-33",
34+
"other-crate-32",
35+
"other-crate-31",
36+
"other-crate-30",
37+
"other-crate-29",
38+
"other-crate-28",
39+
"other-crate-27",
40+
"other-crate-26",
41+
"other-crate-25",
42+
"other-crate-24",
43+
"other-crate-23",
44+
"other-crate-22",
45+
"other-crate-21",
46+
"other-crate-20",
47+
"other-crate-19",
48+
"other-crate-18",
49+
"other-crate-17",
50+
"other-crate-16",
51+
"other-crate-15",
52+
"other-crate-14",
53+
"other-crate-13",
54+
"other-crate-12",
55+
"other-crate-11",
56+
"other-crate-10",
57+
"other-crate-9",
58+
"other-crate-8",
59+
"other-crate-7",
60+
"other-crate-6",
61+
"other-crate-5",
62+
"other-crate-4",
63+
"other-crate-3",
64+
"other-crate-2",
65+
"other-crate-1",
66+
]

‎src/worker/jobs/rss/snapshots/crates_io__worker__jobs__rss__sync_crates_feed__tests__load_version_updates.snap

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
source: src/worker/jobs/rss/sync_crates_feed.rs
3+
expression: "new_crates.iter().map(|u| &u.name).collect::<Vec<_>>()"
4+
---
5+
[
6+
"qux",
7+
"baz",
8+
"bar",
9+
"foo",
10+
]

‎src/worker/jobs/rss/sync_crates_feed.rs

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,23 @@ use crate::schema::crates;
22
use crate::storage::FeedId;
33
use crate::worker::Environment;
44
use anyhow::anyhow;
5+
use chrono::Duration;
56
use crates_io_worker::BackgroundJob;
67
use diesel::prelude::*;
78
use std::sync::Arc;
89

910
#[derive(Serialize, Deserialize)]
1011
pub struct SyncCratesFeed;
1112

13+
/// Items younger than this will always be included in the feed.
14+
const ALWAYS_INCLUDE_AGE: Duration = Duration::minutes(60);
15+
16+
/// The number of items to include in the feed.
17+
///
18+
/// If there are less than this number of items in the database, the feed will
19+
/// contain fewer items. If there are more items in the database that are
20+
/// younger than [`ALWAYS_INCLUDE_AGE`], all of them will be included in
21+
/// the feed.
1222
const NUM_ITEMS: i64 = 50;
1323

1424
impl BackgroundJob for SyncCratesFeed {
@@ -70,7 +80,26 @@ impl BackgroundJob for SyncCratesFeed {
7080
}
7181
}
7282

83+
/// Load the latest crates from the database.
84+
///
85+
/// This function will load all crates from the database that are younger
86+
/// than [`ALWAYS_INCLUDE_AGE`]. If there are less than [`NUM_ITEMS`] crates
87+
/// then the list will be padded with older crates until [`NUM_ITEMS`] are
88+
/// returned.
7389
fn load_new_crates(conn: &mut PgConnection) -> QueryResult<Vec<NewCrate>> {
90+
let threshold_dt = chrono::Utc::now().naive_utc() - ALWAYS_INCLUDE_AGE;
91+
92+
let new_crates = crates::table
93+
.filter(crates::created_at.gt(threshold_dt))
94+
.order(crates::created_at.desc())
95+
.select(NewCrate::as_select())
96+
.load(conn)?;
97+
98+
let num_new_crates = new_crates.len();
99+
if num_new_crates as i64 >= NUM_ITEMS {
100+
return Ok(new_crates);
101+
}
102+
74103
crates::table
75104
.order(crates::created_at.desc())
76105
.select(NewCrate::as_select())
@@ -126,3 +155,68 @@ impl NewCrate {
126155
}
127156
}
128157
}
158+
159+
#[cfg(test)]
160+
mod tests {
161+
use super::*;
162+
use chrono::NaiveDateTime;
163+
use crates_io_test_db::TestDatabase;
164+
use insta::assert_debug_snapshot;
165+
166+
#[test]
167+
fn test_load_version_updates() {
168+
crate::util::tracing::init_for_test();
169+
170+
let db = TestDatabase::new();
171+
let mut conn = db.connect();
172+
173+
let now = chrono::Utc::now().naive_utc();
174+
175+
let new_crates = assert_ok!(load_new_crates(&mut conn));
176+
assert_eq!(new_crates.len(), 0);
177+
178+
// If there are less than NUM_ITEMS crates, they should all be returned
179+
create_crate(&mut conn, "foo", now - Duration::days(123));
180+
create_crate(&mut conn, "bar", now - Duration::days(110));
181+
create_crate(&mut conn, "baz", now - Duration::days(100));
182+
create_crate(&mut conn, "qux", now - Duration::days(90));
183+
184+
let new_crates = assert_ok!(load_new_crates(&mut conn));
185+
assert_eq!(new_crates.len(), 4);
186+
assert_debug_snapshot!(new_crates.iter().map(|u| &u.name).collect::<Vec<_>>());
187+
188+
// If there are more than NUM_ITEMS crates, only the most recent NUM_ITEMS should be returned
189+
for i in 1..=NUM_ITEMS {
190+
let name = format!("crate-{i}");
191+
let publish_time = now - Duration::days(90) + Duration::hours(i);
192+
create_crate(&mut conn, &name, publish_time);
193+
}
194+
195+
let new_crates = assert_ok!(load_new_crates(&mut conn));
196+
assert_eq!(new_crates.len() as i64, NUM_ITEMS);
197+
assert_debug_snapshot!(new_crates.iter().map(|u| &u.name).collect::<Vec<_>>());
198+
199+
// But if there are more than NUM_ITEMS crates that are younger than ALWAYS_INCLUDE_AGE, all of them should be returned
200+
for i in 1..=(NUM_ITEMS + 10) {
201+
let name = format!("other-crate-{i}");
202+
let publish_time = now - Duration::minutes(30) + Duration::seconds(i);
203+
create_crate(&mut conn, &name, publish_time);
204+
}
205+
206+
let new_crates = assert_ok!(load_new_crates(&mut conn));
207+
assert_eq!(new_crates.len() as i64, NUM_ITEMS + 10);
208+
assert_debug_snapshot!(new_crates.iter().map(|u| &u.name).collect::<Vec<_>>());
209+
}
210+
211+
fn create_crate(conn: &mut PgConnection, name: &str, publish_time: NaiveDateTime) -> i32 {
212+
diesel::insert_into(crates::table)
213+
.values((
214+
crates::name.eq(name),
215+
crates::created_at.eq(publish_time),
216+
crates::updated_at.eq(publish_time),
217+
))
218+
.returning(crates::id)
219+
.get_result(conn)
220+
.unwrap()
221+
}
222+
}

0 commit comments

Comments
 (0)