Skip to content

Change how errors are stored in the DB #1641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 16, 2023
Merged

Change how errors are stored in the DB #1641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3c758be
Change how errors are stored in the DB
Kobzol Jul 6, 2023
df61f2a
Fix SQLite/Postgres conversion binaries
Kobzol Jul 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 2 additions & 25 deletions database/src/bin/postgres-to-sqlite.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,12 +191,12 @@ impl Table for Error {
}

fn postgres_select_statement(&self, since_weeks_ago: Option<u32>) -> String {
let s = "select series, aid, error from ".to_string() + self.name();
let s = "select benchmark, aid, error from ".to_string() + self.name();
with_filter_clause_maybe(s, ARTIFACT_JOIN_AND_WHERE, since_weeks_ago)
}

fn sqlite_insert_statement(&self) -> &'static str {
"insert into error (series, aid, error) VALUES (?, ?, ?)"
"insert into error (benchmark, aid, error) VALUES (?, ?, ?)"
}

fn sqlite_execute_insert(&self, statement: &mut rusqlite::Statement, row: tokio_postgres::Row) {
Expand All @@ -210,28 +210,6 @@ impl Table for Error {
}
}

struct ErrorSeries;

impl Table for ErrorSeries {
fn name(&self) -> &'static str {
"error_series"
}

fn postgres_select_statement(&self, _since_weeks_ago: Option<u32>) -> String {
"select id, crate from ".to_string() + self.name()
}

fn sqlite_insert_statement(&self) -> &'static str {
"insert into error_series (id, crate) VALUES (?, ?)"
}

fn sqlite_execute_insert(&self, statement: &mut rusqlite::Statement, row: tokio_postgres::Row) {
statement
.execute(params![row.get::<_, i32>(0), row.get::<_, &str>(1)])
.unwrap();
}
}

struct Pstat;

impl Table for Pstat {
Expand Down Expand Up @@ -450,7 +428,6 @@ async fn main() -> anyhow::Result<()> {
&Benchmark,
&Collection,
&CollectorProgress,
&ErrorSeries,
&Error,
&PstatSeries,
&Pstat,
Expand Down
38 changes: 3 additions & 35 deletions database/src/bin/sqlite-to-postgres.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ struct Error;

#[derive(Serialize)]
struct ErrorRow<'a> {
series: i32,
benchmark: &'a str,
aid: i32,
error: Nullable<&'a str>,
}
Expand All @@ -229,7 +229,7 @@ impl Table for Error {
}

fn sqlite_attributes() -> &'static str {
"series, aid, error"
"benchmark, aid, error"
}

fn postgres_generated_id_attribute() -> Option<&'static str> {
Expand All @@ -239,45 +239,14 @@ impl Table for Error {
fn write_postgres_csv_row<W: Write>(writer: &mut csv::Writer<W>, row: &rusqlite::Row) {
writer
.serialize(ErrorRow {
series: row.get(0).unwrap(),
benchmark: row.get_ref(0).unwrap().as_str().unwrap(),
aid: row.get(1).unwrap(),
error: row.get_ref(2).unwrap().try_into().unwrap(),
})
.unwrap();
}
}

struct ErrorSeries;

#[derive(Serialize)]
struct ErrorSeriesRow<'a> {
id: i32,
krate: &'a str,
}

impl Table for ErrorSeries {
fn name() -> &'static str {
"error_series"
}

fn sqlite_attributes() -> &'static str {
"id, crate"
}

fn postgres_generated_id_attribute() -> Option<&'static str> {
Some("id")
}

fn write_postgres_csv_row<W: Write>(writer: &mut csv::Writer<W>, row: &rusqlite::Row) {
writer
.serialize(ErrorSeriesRow {
id: row.get(0).unwrap(),
krate: row.get_ref(1).unwrap().as_str().unwrap(),
})
.unwrap();
}
}

struct Pstat;

#[derive(Serialize)]
Expand Down Expand Up @@ -664,7 +633,6 @@ async fn main() -> anyhow::Result<()> {
copy::<Benchmark>(&sqlite_tx, &postgres_tx).await;
copy::<Collection>(&sqlite_tx, &postgres_tx).await;
copy::<CollectorProgress>(&sqlite_tx, &postgres_tx).await;
copy::<ErrorSeries>(&sqlite_tx, &postgres_tx).await;
copy::<Error>(&sqlite_tx, &postgres_tx).await;
copy::<PstatSeries>(&sqlite_tx, &postgres_tx).await;
copy::<Pstat>(&sqlite_tx, &postgres_tx).await;
Expand Down
58 changes: 28 additions & 30 deletions database/src/pool/postgres.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,22 @@ static MIGRATIONS: &[&str] = &[
value double precision not null,
PRIMARY KEY(series, aid, cid)
);
"#,
r#"
create table error_new(
aid integer not null references artifact(id) on delete cascade on update cascade,
benchmark text not null,
error text not null,
primary key(aid, benchmark)
);
insert into error_new(aid, benchmark, error)
select aid, crate, error
from error
join error_series es on error.series = es.id;

drop table error;
drop table error_series;
alter table error_new rename to error;
"#,
];

Expand Down Expand Up @@ -368,7 +384,7 @@ impl PostgresConnection {
PostgresConnection {
statements: Arc::new(CachedStatements {
get_pstat: conn
.prepare("
.prepare("
WITH aids AS (
select aid, num from unnest($2::int[]) with ordinality aids(aid, num)
),
Expand Down Expand Up @@ -432,8 +448,7 @@ impl PostgresConnection {
)
.await
.unwrap(),
get_error: conn.prepare("select crate, error from error_series
inner join error on error.series = error_series.id and aid = $1").await.unwrap(),
get_error: conn.prepare("select benchmark, error from error where aid = $1").await.unwrap(),
select_self_query_series: conn.prepare("select id from self_profile_query_series where crate = $1 and profile = $2 and cache = $3 and query = $4").await.unwrap(),
insert_self_query_series: conn.prepare("insert into self_profile_query_series (crate, profile, cache, query) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, cache, statistic) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
Expand Down Expand Up @@ -700,7 +715,7 @@ where
&[&(pr as i32), &include, &exclude, &runs],
)
.await {
log::error!("failed to queue_pr({}, {:?}, {:?}, {:?}): {:?}", pr, include, exclude, runs, e);
log::error!("failed to queue_pr({}, {:?}, {:?}, {:?}): {:?}", pr, include, exclude, runs, e);
}
}
async fn pr_attach_commit(
Expand Down Expand Up @@ -907,13 +922,13 @@ where
Some(aid) => aid.get::<_, i32>(0) as u32,
None => {
self.conn()
.query_opt("insert into artifact (name, date, type) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING RETURNING id", &[
&name,
&date,
&ty,
])
.await
.unwrap();
.query_opt("insert into artifact (name, date, type) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING RETURNING id", &[
&name,
&date,
&ty,
])
.await
.unwrap();
self.conn()
.query_one("select id from artifact where name = $1", &[&name])
.await
Expand Down Expand Up @@ -984,27 +999,10 @@ where
}

async fn record_error(&self, artifact: ArtifactIdNumber, krate: &str, error: &str) {
let sid = self
.conn()
.query_opt(
"insert into error_series (crate) VALUES ($1) ON CONFLICT DO NOTHING RETURNING id",
&[&krate],
)
.await
.unwrap();
let sid: i32 = match sid {
Some(id) => id.get(0),
None => self
.conn()
.query_one("select id from error_series where crate = $1", &[&krate])
.await
.unwrap()
.get(0),
};
self.conn()
.execute(
"insert into error (series, aid, error) VALUES ($1, $2, $3)",
&[&sid, &(artifact.0 as i32), &error],
"insert into error (benchmark, aid, error) VALUES ($1, $2, $3)",
&[&krate, &(artifact.0 as i32), &error],
)
.await
.unwrap();
Expand Down
41 changes: 21 additions & 20 deletions database/src/pool/sqlite.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,24 @@ static MIGRATIONS: &[Migration] = &[
);
"#,
),
Migration::new(
r#"
create table error_new(
aid integer references artifact(id) on delete cascade on update cascade,
benchmark text not null,
error text not null,
primary key(aid, benchmark)
);
insert into error_new(aid, benchmark, error)
select aid, crate, error
from error
join error_series es on error.series = es.id;

drop table error;
drop table error_series;
alter table error_new rename to error;
"#,
),
];

#[async_trait::async_trait]
Expand Down Expand Up @@ -748,22 +766,8 @@ impl Connection for SqliteConnection {
async fn record_error(&self, artifact: ArtifactIdNumber, krate: &str, error: &str) {
self.raw_ref()
.execute(
"insert or ignore into error_series (crate) VALUES (?)",
params![&krate,],
)
.unwrap();
let sid: i32 = self
.raw_ref()
.query_row(
"select id from error_series where crate = ?",
params![&krate,],
|r| r.get(0),
)
.unwrap();
self.raw_ref()
.execute(
"insert into error (series, aid, error) VALUES (?, ?, ?)",
params![&sid, &artifact.0, &error],
"insert into error (benchmark, aid, error) VALUES (?, ?, ?)",
params![krate, &artifact.0, &error],
)
.unwrap();
}
Expand Down Expand Up @@ -907,10 +911,7 @@ impl Connection for SqliteConnection {
}
async fn get_error(&self, aid: crate::ArtifactIdNumber) -> HashMap<String, String> {
self.raw_ref()
.prepare_cached(
"select crate, error from error_series
inner join error on error.series = error_series.id and aid = ?",
)
.prepare_cached("select benchmark, error from error where aid = ?")
.unwrap()
.query_map(params![&aid.0], |row| Ok((row.get(0)?, row.get(1)?)))
.unwrap()
Expand Down