Skip to content

Commit ff7901a

Browse files
Turbo87Turbo87
committed
rss: Always include *all* new crates from the past 60 minutes
1 parent 1e6881b commit ff7901a

4 files changed

+226
-0
lines changed

‎src/worker/jobs/rss/snapshots/crates_io__worker__jobs__rss__sync_crates_feed__tests__load_version_updates-2.snap

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
---
2+
source: src/worker/jobs/rss/sync_crates_feed.rs
3+
expression: "new_crates.iter().map(|u| &u.name).collect::<Vec<_>>()"
4+
---
5+
[
6+
"crate-50",
7+
"crate-49",
8+
"crate-48",
9+
"crate-47",
10+
"crate-46",
11+
"crate-45",
12+
"crate-44",
13+
"crate-43",
14+
"crate-42",
15+
"crate-41",
16+
"crate-40",
17+
"crate-39",
18+
"crate-38",
19+
"crate-37",
20+
"crate-36",
21+
"crate-35",
22+
"crate-34",
23+
"crate-33",
24+
"crate-32",
25+
"crate-31",
26+
"crate-30",
27+
"crate-29",
28+
"crate-28",
29+
"crate-27",
30+
"crate-26",
31+
"crate-25",
32+
"crate-24",
33+
"crate-23",
34+
"crate-22",
35+
"crate-21",
36+
"crate-20",
37+
"crate-19",
38+
"crate-18",
39+
"crate-17",
40+
"crate-16",
41+
"crate-15",
42+
"crate-14",
43+
"crate-13",
44+
"crate-12",
45+
"crate-11",
46+
"crate-10",
47+
"crate-9",
48+
"crate-8",
49+
"crate-7",
50+
"crate-6",
51+
"crate-5",
52+
"crate-4",
53+
"crate-3",
54+
"crate-2",
55+
"crate-1",
56+
]

‎src/worker/jobs/rss/snapshots/crates_io__worker__jobs__rss__sync_crates_feed__tests__load_version_updates-3.snap

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
---
2+
source: src/worker/jobs/rss/sync_crates_feed.rs
3+
expression: "new_crates.iter().map(|u| &u.name).collect::<Vec<_>>()"
4+
---
5+
[
6+
"other-crate-60",
7+
"other-crate-59",
8+
"other-crate-58",
9+
"other-crate-57",
10+
"other-crate-56",
11+
"other-crate-55",
12+
"other-crate-54",
13+
"other-crate-53",
14+
"other-crate-52",
15+
"other-crate-51",
16+
"other-crate-50",
17+
"other-crate-49",
18+
"other-crate-48",
19+
"other-crate-47",
20+
"other-crate-46",
21+
"other-crate-45",
22+
"other-crate-44",
23+
"other-crate-43",
24+
"other-crate-42",
25+
"other-crate-41",
26+
"other-crate-40",
27+
"other-crate-39",
28+
"other-crate-38",
29+
"other-crate-37",
30+
"other-crate-36",
31+
"other-crate-35",
32+
"other-crate-34",
33+
"other-crate-33",
34+
"other-crate-32",
35+
"other-crate-31",
36+
"other-crate-30",
37+
"other-crate-29",
38+
"other-crate-28",
39+
"other-crate-27",
40+
"other-crate-26",
41+
"other-crate-25",
42+
"other-crate-24",
43+
"other-crate-23",
44+
"other-crate-22",
45+
"other-crate-21",
46+
"other-crate-20",
47+
"other-crate-19",
48+
"other-crate-18",
49+
"other-crate-17",
50+
"other-crate-16",
51+
"other-crate-15",
52+
"other-crate-14",
53+
"other-crate-13",
54+
"other-crate-12",
55+
"other-crate-11",
56+
"other-crate-10",
57+
"other-crate-9",
58+
"other-crate-8",
59+
"other-crate-7",
60+
"other-crate-6",
61+
"other-crate-5",
62+
"other-crate-4",
63+
"other-crate-3",
64+
"other-crate-2",
65+
"other-crate-1",
66+
]

‎src/worker/jobs/rss/snapshots/crates_io__worker__jobs__rss__sync_crates_feed__tests__load_version_updates.snap

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
source: src/worker/jobs/rss/sync_crates_feed.rs
3+
expression: "new_crates.iter().map(|u| &u.name).collect::<Vec<_>>()"
4+
---
5+
[
6+
"qux",
7+
"baz",
8+
"bar",
9+
"foo",
10+
]

‎src/worker/jobs/rss/sync_crates_feed.rs

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,23 @@ use crate::schema::crates;
22
use crate::storage::FeedId;
33
use crate::worker::Environment;
44
use anyhow::anyhow;
5+
use chrono::Duration;
56
use crates_io_worker::BackgroundJob;
67
use diesel::prelude::*;
78
use std::sync::Arc;
89

910
#[derive(Serialize, Deserialize)]
1011
pub struct SyncCratesFeed;
1112

13+
/// Items younger than this will always be included in the feed.
14+
const ALWAYS_INCLUDE_AGE: Duration = Duration::minutes(60);
15+
16+
/// The number of items to include in the feed.
17+
///
18+
/// If there are less than this number of items in the database, the feed will
19+
/// contain fewer items. If there are more items in the database that are
20+
/// younger than [`ALWAYS_INCLUDE_AGE`], all of them will be included in
21+
/// the feed.
1222
const NUM_ITEMS: i64 = 50;
1323

1424
impl BackgroundJob for SyncCratesFeed {
@@ -76,7 +86,26 @@ impl BackgroundJob for SyncCratesFeed {
7686
}
7787
}
7888

89+
/// Load the latest crates from the database.
90+
///
91+
/// This function will load all crates from the database that are younger
92+
/// than [`ALWAYS_INCLUDE_AGE`]. If there are less than [`NUM_ITEMS`] crates
93+
/// then the list will be padded with older crates until [`NUM_ITEMS`] are
94+
/// returned.
7995
fn load_new_crates(conn: &mut PgConnection) -> QueryResult<Vec<NewCrate>> {
96+
let threshold_dt = chrono::Utc::now().naive_utc() - ALWAYS_INCLUDE_AGE;
97+
98+
let new_crates = crates::table
99+
.filter(crates::created_at.gt(threshold_dt))
100+
.order(crates::created_at.desc())
101+
.select(NewCrate::as_select())
102+
.load(conn)?;
103+
104+
let num_new_crates = new_crates.len();
105+
if num_new_crates as i64 >= NUM_ITEMS {
106+
return Ok(new_crates);
107+
}
108+
80109
crates::table
81110
.order(crates::created_at.desc())
82111
.select(NewCrate::as_select())
@@ -132,3 +161,68 @@ impl NewCrate {
132161
}
133162
}
134163
}
164+
165+
#[cfg(test)]
166+
mod tests {
167+
use super::*;
168+
use chrono::NaiveDateTime;
169+
use crates_io_test_db::TestDatabase;
170+
use insta::assert_debug_snapshot;
171+
172+
#[test]
173+
fn test_load_version_updates() {
174+
crate::util::tracing::init_for_test();
175+
176+
let db = TestDatabase::new();
177+
let mut conn = db.connect();
178+
179+
let now = chrono::Utc::now().naive_utc();
180+
181+
let new_crates = assert_ok!(load_new_crates(&mut conn));
182+
assert_eq!(new_crates.len(), 0);
183+
184+
// If there are less than NUM_ITEMS crates, they should all be returned
185+
create_crate(&mut conn, "foo", now - Duration::days(123));
186+
create_crate(&mut conn, "bar", now - Duration::days(110));
187+
create_crate(&mut conn, "baz", now - Duration::days(100));
188+
create_crate(&mut conn, "qux", now - Duration::days(90));
189+
190+
let new_crates = assert_ok!(load_new_crates(&mut conn));
191+
assert_eq!(new_crates.len(), 4);
192+
assert_debug_snapshot!(new_crates.iter().map(|u| &u.name).collect::<Vec<_>>());
193+
194+
// If there are more than NUM_ITEMS crates, only the most recent NUM_ITEMS should be returned
195+
for i in 1..=NUM_ITEMS {
196+
let name = format!("crate-{i}");
197+
let publish_time = now - Duration::days(90) + Duration::hours(i);
198+
create_crate(&mut conn, &name, publish_time);
199+
}
200+
201+
let new_crates = assert_ok!(load_new_crates(&mut conn));
202+
assert_eq!(new_crates.len() as i64, NUM_ITEMS);
203+
assert_debug_snapshot!(new_crates.iter().map(|u| &u.name).collect::<Vec<_>>());
204+
205+
// But if there are more than NUM_ITEMS crates that are younger than ALWAYS_INCLUDE_AGE, all of them should be returned
206+
for i in 1..=(NUM_ITEMS + 10) {
207+
let name = format!("other-crate-{i}");
208+
let publish_time = now - Duration::minutes(30) + Duration::seconds(i);
209+
create_crate(&mut conn, &name, publish_time);
210+
}
211+
212+
let new_crates = assert_ok!(load_new_crates(&mut conn));
213+
assert_eq!(new_crates.len() as i64, NUM_ITEMS + 10);
214+
assert_debug_snapshot!(new_crates.iter().map(|u| &u.name).collect::<Vec<_>>());
215+
}
216+
217+
fn create_crate(conn: &mut PgConnection, name: &str, publish_time: NaiveDateTime) -> i32 {
218+
diesel::insert_into(crates::table)
219+
.values((
220+
crates::name.eq(name),
221+
crates::created_at.eq(publish_time),
222+
crates::updated_at.eq(publish_time),
223+
))
224+
.returning(crates::id)
225+
.get_result(conn)
226+
.unwrap()
227+
}
228+
}

0 commit comments

Comments
 (0)