Skip to content

Commit 06c13c1

Browse files
Turbo87Turbo87
committed
rss: Always include *all* version updates from the past 60 minutes
1 parent 3eb969f commit 06c13c1

4 files changed

+344
-0
lines changed

‎src/worker/jobs/rss/snapshots/crates_io__worker__jobs__rss__sync_updates_feed__tests__load_version_updates-2.snap

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
---
2+
source: src/worker/jobs/rss/sync_updates_feed.rs
3+
expression: "updates.iter().map(|u| &u.version).collect::<Vec<_>>()"
4+
---
5+
[
6+
"1.2.100",
7+
"1.2.99",
8+
"1.2.98",
9+
"1.2.97",
10+
"1.2.96",
11+
"1.2.95",
12+
"1.2.94",
13+
"1.2.93",
14+
"1.2.92",
15+
"1.2.91",
16+
"1.2.90",
17+
"1.2.89",
18+
"1.2.88",
19+
"1.2.87",
20+
"1.2.86",
21+
"1.2.85",
22+
"1.2.84",
23+
"1.2.83",
24+
"1.2.82",
25+
"1.2.81",
26+
"1.2.80",
27+
"1.2.79",
28+
"1.2.78",
29+
"1.2.77",
30+
"1.2.76",
31+
"1.2.75",
32+
"1.2.74",
33+
"1.2.73",
34+
"1.2.72",
35+
"1.2.71",
36+
"1.2.70",
37+
"1.2.69",
38+
"1.2.68",
39+
"1.2.67",
40+
"1.2.66",
41+
"1.2.65",
42+
"1.2.64",
43+
"1.2.63",
44+
"1.2.62",
45+
"1.2.61",
46+
"1.2.60",
47+
"1.2.59",
48+
"1.2.58",
49+
"1.2.57",
50+
"1.2.56",
51+
"1.2.55",
52+
"1.2.54",
53+
"1.2.53",
54+
"1.2.52",
55+
"1.2.51",
56+
"1.2.50",
57+
"1.2.49",
58+
"1.2.48",
59+
"1.2.47",
60+
"1.2.46",
61+
"1.2.45",
62+
"1.2.44",
63+
"1.2.43",
64+
"1.2.42",
65+
"1.2.41",
66+
"1.2.40",
67+
"1.2.39",
68+
"1.2.38",
69+
"1.2.37",
70+
"1.2.36",
71+
"1.2.35",
72+
"1.2.34",
73+
"1.2.33",
74+
"1.2.32",
75+
"1.2.31",
76+
"1.2.30",
77+
"1.2.29",
78+
"1.2.28",
79+
"1.2.27",
80+
"1.2.26",
81+
"1.2.25",
82+
"1.2.24",
83+
"1.2.23",
84+
"1.2.22",
85+
"1.2.21",
86+
"1.2.20",
87+
"1.2.19",
88+
"1.2.18",
89+
"1.2.17",
90+
"1.2.16",
91+
"1.2.15",
92+
"1.2.14",
93+
"1.2.13",
94+
"1.2.12",
95+
"1.2.11",
96+
"1.2.10",
97+
"1.2.9",
98+
"1.2.8",
99+
"1.2.7",
100+
"1.2.6",
101+
"1.2.5",
102+
"1.2.4",
103+
"1.2.3",
104+
"1.2.2",
105+
"1.2.1",
106+
]

‎src/worker/jobs/rss/snapshots/crates_io__worker__jobs__rss__sync_updates_feed__tests__load_version_updates-3.snap

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
---
2+
source: src/worker/jobs/rss/sync_updates_feed.rs
3+
expression: "updates.iter().map(|u| &u.version).collect::<Vec<_>>()"
4+
---
5+
[
6+
"1.3.110",
7+
"1.3.109",
8+
"1.3.108",
9+
"1.3.107",
10+
"1.3.106",
11+
"1.3.105",
12+
"1.3.104",
13+
"1.3.103",
14+
"1.3.102",
15+
"1.3.101",
16+
"1.3.100",
17+
"1.3.99",
18+
"1.3.98",
19+
"1.3.97",
20+
"1.3.96",
21+
"1.3.95",
22+
"1.3.94",
23+
"1.3.93",
24+
"1.3.92",
25+
"1.3.91",
26+
"1.3.90",
27+
"1.3.89",
28+
"1.3.88",
29+
"1.3.87",
30+
"1.3.86",
31+
"1.3.85",
32+
"1.3.84",
33+
"1.3.83",
34+
"1.3.82",
35+
"1.3.81",
36+
"1.3.80",
37+
"1.3.79",
38+
"1.3.78",
39+
"1.3.77",
40+
"1.3.76",
41+
"1.3.75",
42+
"1.3.74",
43+
"1.3.73",
44+
"1.3.72",
45+
"1.3.71",
46+
"1.3.70",
47+
"1.3.69",
48+
"1.3.68",
49+
"1.3.67",
50+
"1.3.66",
51+
"1.3.65",
52+
"1.3.64",
53+
"1.3.63",
54+
"1.3.62",
55+
"1.3.61",
56+
"1.3.60",
57+
"1.3.59",
58+
"1.3.58",
59+
"1.3.57",
60+
"1.3.56",
61+
"1.3.55",
62+
"1.3.54",
63+
"1.3.53",
64+
"1.3.52",
65+
"1.3.51",
66+
"1.3.50",
67+
"1.3.49",
68+
"1.3.48",
69+
"1.3.47",
70+
"1.3.46",
71+
"1.3.45",
72+
"1.3.44",
73+
"1.3.43",
74+
"1.3.42",
75+
"1.3.41",
76+
"1.3.40",
77+
"1.3.39",
78+
"1.3.38",
79+
"1.3.37",
80+
"1.3.36",
81+
"1.3.35",
82+
"1.3.34",
83+
"1.3.33",
84+
"1.3.32",
85+
"1.3.31",
86+
"1.3.30",
87+
"1.3.29",
88+
"1.3.28",
89+
"1.3.27",
90+
"1.3.26",
91+
"1.3.25",
92+
"1.3.24",
93+
"1.3.23",
94+
"1.3.22",
95+
"1.3.21",
96+
"1.3.20",
97+
"1.3.19",
98+
"1.3.18",
99+
"1.3.17",
100+
"1.3.16",
101+
"1.3.15",
102+
"1.3.14",
103+
"1.3.13",
104+
"1.3.12",
105+
"1.3.11",
106+
"1.3.10",
107+
"1.3.9",
108+
"1.3.8",
109+
"1.3.7",
110+
"1.3.6",
111+
"1.3.5",
112+
"1.3.4",
113+
"1.3.3",
114+
"1.3.2",
115+
"1.3.1",
116+
]

‎src/worker/jobs/rss/snapshots/crates_io__worker__jobs__rss__sync_updates_feed__tests__load_version_updates.snap

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
source: src/worker/jobs/rss/sync_updates_feed.rs
3+
expression: "updates.iter().map(|u| &u.version).collect::<Vec<_>>()"
4+
---
5+
[
6+
"1.2.0",
7+
"1.1.0",
8+
"1.0.1",
9+
"1.0.0",
10+
]

‎src/worker/jobs/rss/sync_updates_feed.rs

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,23 @@ use crate::schema::{crates, versions};
22
use crate::storage::FeedId;
33
use crate::worker::Environment;
44
use anyhow::anyhow;
5+
use chrono::Duration;
56
use crates_io_worker::BackgroundJob;
67
use diesel::prelude::*;
78
use std::sync::Arc;
89

910
#[derive(Serialize, Deserialize)]
1011
pub struct SyncUpdatesFeed;
1112

13+
/// Items younger than this will always be included in the feed.
14+
const ALWAYS_INCLUDE_AGE: Duration = Duration::minutes(60);
15+
16+
/// The number of items to include in the feed.
17+
///
18+
/// If there are less than this number of items in the database, the feed will
19+
/// contain fewer items. If there are more items in the database that are
20+
/// younger than [`ALWAYS_INCLUDE_AGE`], all of them will be included in
21+
/// the feed.
1222
const NUM_ITEMS: i64 = 100;
1323

1424
impl BackgroundJob for SyncUpdatesFeed {
@@ -70,7 +80,27 @@ impl BackgroundJob for SyncUpdatesFeed {
7080
}
7181
}
7282

83+
/// Load the latest versions from the database.
84+
///
85+
/// This function will load all versions from the database that are younger
86+
/// than [`ALWAYS_INCLUDE_AGE`]. If there are less than [`NUM_ITEMS`] versions
87+
/// then the list will be padded with older versions until [`NUM_ITEMS`] are
88+
/// returned.
7389
fn load_version_updates(conn: &mut PgConnection) -> QueryResult<Vec<VersionUpdate>> {
90+
let threshold_dt = chrono::Utc::now().naive_utc() - ALWAYS_INCLUDE_AGE;
91+
92+
let updates = versions::table
93+
.inner_join(crates::table)
94+
.filter(versions::created_at.gt(threshold_dt))
95+
.order(versions::created_at.desc())
96+
.select(VersionUpdate::as_select())
97+
.load(conn)?;
98+
99+
let num_updates = updates.len();
100+
if num_updates as i64 >= NUM_ITEMS {
101+
return Ok(updates);
102+
}
103+
74104
versions::table
75105
.inner_join(crates::table)
76106
.order(versions::created_at.desc())
@@ -141,3 +171,85 @@ impl VersionUpdate {
141171
}
142172
}
143173
}
174+
175+
#[cfg(test)]
176+
mod tests {
177+
use super::*;
178+
use chrono::NaiveDateTime;
179+
use crates_io_test_db::TestDatabase;
180+
use insta::assert_debug_snapshot;
181+
182+
#[test]
183+
fn test_load_version_updates() {
184+
crate::util::tracing::init_for_test();
185+
186+
let db = TestDatabase::new();
187+
let mut conn = db.connect();
188+
189+
let now = chrono::Utc::now().naive_utc();
190+
191+
let updates = assert_ok!(load_version_updates(&mut conn));
192+
assert_eq!(updates.len(), 0);
193+
194+
let foo = create_crate(&mut conn, "foo");
195+
196+
// If there are less than NUM_ITEMS versions, they should all be returned
197+
create_version(&mut conn, foo, "1.0.0", now - Duration::days(123));
198+
create_version(&mut conn, foo, "1.0.1", now - Duration::days(110));
199+
create_version(&mut conn, foo, "1.1.0", now - Duration::days(100));
200+
create_version(&mut conn, foo, "1.2.0", now - Duration::days(90));
201+
202+
let updates = assert_ok!(load_version_updates(&mut conn));
203+
assert_eq!(updates.len(), 4);
204+
assert_debug_snapshot!(updates.iter().map(|u| &u.version).collect::<Vec<_>>());
205+
206+
// If there are more than NUM_ITEMS versions, only the most recent NUM_ITEMS should be returned
207+
for i in 1..=NUM_ITEMS {
208+
let version = format!("1.2.{i}");
209+
let publish_time = now - Duration::days(90) + Duration::hours(i);
210+
create_version(&mut conn, foo, &version, publish_time);
211+
}
212+
213+
let updates = assert_ok!(load_version_updates(&mut conn));
214+
assert_eq!(updates.len() as i64, NUM_ITEMS);
215+
assert_debug_snapshot!(updates.iter().map(|u| &u.version).collect::<Vec<_>>());
216+
217+
// But if there are more than NUM_ITEMS versions that are younger than ALWAYS_INCLUDE_AGE, all of them should be returned
218+
for i in 1..=(NUM_ITEMS + 10) {
219+
let version = format!("1.3.{i}");
220+
let publish_time = now - Duration::minutes(30) + Duration::seconds(i);
221+
create_version(&mut conn, foo, &version, publish_time);
222+
}
223+
224+
let updates = assert_ok!(load_version_updates(&mut conn));
225+
assert_eq!(updates.len() as i64, NUM_ITEMS + 10);
226+
assert_debug_snapshot!(updates.iter().map(|u| &u.version).collect::<Vec<_>>());
227+
}
228+
229+
fn create_crate(conn: &mut PgConnection, name: &str) -> i32 {
230+
diesel::insert_into(crates::table)
231+
.values((crates::name.eq(name),))
232+
.returning(crates::id)
233+
.get_result(conn)
234+
.unwrap()
235+
}
236+
237+
fn create_version(
238+
conn: &mut PgConnection,
239+
crate_id: i32,
240+
version: &str,
241+
publish_time: NaiveDateTime,
242+
) -> i32 {
243+
diesel::insert_into(versions::table)
244+
.values((
245+
versions::crate_id.eq(crate_id),
246+
versions::num.eq(version),
247+
versions::created_at.eq(publish_time),
248+
versions::updated_at.eq(publish_time),
249+
versions::checksum.eq("checksum"),
250+
))
251+
.returning(versions::id)
252+
.get_result(conn)
253+
.unwrap()
254+
}
255+
}

0 commit comments

Comments
 (0)