Skip to content

lintcheck: support git sources #6686

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Feb 7, 2021
Merged

lintcheck: support git sources #6686

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cb30219
lintcheck: update logs
matthiaskrgr Feb 6, 2021
64982cc
lintcheck: make TomlCrate also accept git-data from lintcheck_crates.…
matthiaskrgr Feb 5, 2021
10fbafa
implement the download_and_extract() step for git sources
matthiaskrgr Feb 6, 2021
9ab505a
lintcheck: add git source as an example and update logs
matthiaskrgr Feb 6, 2021
e1c284b
lintcheck: cleanup, fix --only for git crates, better error msgs
matthiaskrgr Feb 6, 2021
2bffbfc
lintcheck: avoid dbg!() calls
matthiaskrgr Feb 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 18 additions & 17 deletions clippy_dev/lintcheck_crates.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
[crates]
# some of these are from cargotest
cargo = ['0.49.0']
iron = ['0.6.1']
ripgrep = ['12.1.1']
xsv = ['0.13.0']
#tokei = ['12.0.4']
rayon = ['1.5.0']
serde = ['1.0.118']
cargo = {name = "cargo", versions = ['0.49.0']}
iron = {name = "iron", versions = ['0.6.1']}
ripgrep = {name = "ripgrep", versions = ['12.1.1']}
xsv = {name = "xsv", versions = ['0.13.0']}
#tokei = { name = "tokei", versions = ['12.0.4']}
rayon = {name = "rayon", versions = ['1.5.0']}
serde = {name = "serde", versions = ['1.0.118']}
# top 10 crates.io dls
bitflags = ['1.2.1']
libc = ['0.2.81']
log = ['0.4.11']
proc-macro2 = ['1.0.24']
quote = ['1.0.7']
rand = ['0.7.3']
rand_core = ['0.6.0']
regex = ['1.3.2']
syn = ['1.0.54']
unicode-xid = ['0.2.1']
bitflags = {name = "bitflags", versions = ['1.2.1']}
libc = {name = "libc", versions = ['0.2.81']}
log = {name = "log", versions = ['0.4.11']}
proc-macro2 = {name = "proc-macro2", versions = ['1.0.24']}
puffin = {name = "puffin", git_url = "https://github.com/EmbarkStudios/puffin", git_hash = "02dd4a3"}
quote = {name = "quote", versions = ['1.0.7']}
rand = {name = "rand", versions = ['0.7.3']}
rand_core = {name = "rand_core", versions = ['0.6.0']}
regex = {name = "regex", versions = ['1.3.2']}
syn = {name = "syn", versions = ['1.0.54']}
unicode-xid = {name = "unicode-xid", versions = ['0.2.1']}
167 changes: 118 additions & 49 deletions clippy_dev/src/lintcheck.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,29 @@ use serde_json::Value;
// use this to store the crates when interacting with the crates.toml file
#[derive(Debug, Serialize, Deserialize)]
struct CrateList {
crates: HashMap<String, Vec<String>>,
crates: HashMap<String, TomlCrate>,
}

// crate data we stored in the toml, can have multiple versions per crate
// A single TomlCrate is laster mapped to several CrateSources in that case
#[derive(Debug, Serialize, Deserialize)]
struct TomlCrate {
name: String,
versions: Vec<String>,
versions: Option<Vec<String>>,
git_url: Option<String>,
git_hash: Option<String>,
}

// represents an archive we download from crates.io
#[derive(Debug, Serialize, Deserialize, Eq, Hash, PartialEq)]
struct CrateSource {
name: String,
version: String,
enum CrateSource {
CratesIo { name: String, version: String },
Git { name: String, url: String, commit: String },
}

// represents the extracted sourcecode of a crate
// we actually don't need to special-case git repos here because it does not matter for clippy, yay!
// (clippy only needs a simple path)
#[derive(Debug)]
struct Crate {
version: String,
Expand Down Expand Up @@ -69,40 +74,71 @@ impl std::fmt::Display for ClippyWarning {

impl CrateSource {
fn download_and_extract(&self) -> Crate {
let extract_dir = PathBuf::from("target/lintcheck/crates");
let krate_download_dir = PathBuf::from("target/lintcheck/downloads");

// url to download the crate from crates.io
let url = format!(
"https://crates.io/api/v1/crates/{}/{}/download",
self.name, self.version
);
println!("Downloading and extracting {} {} from {}", self.name, self.version, url);
let _ = std::fs::create_dir("target/lintcheck/");
let _ = std::fs::create_dir(&krate_download_dir);
let _ = std::fs::create_dir(&extract_dir);

let krate_file_path = krate_download_dir.join(format!("{}-{}.crate.tar.gz", &self.name, &self.version));
// don't download/extract if we already have done so
if !krate_file_path.is_file() {
// create a file path to download and write the crate data into
let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap();
let mut krate_req = ureq::get(&url).call().unwrap().into_reader();
// copy the crate into the file
std::io::copy(&mut krate_req, &mut krate_dest).unwrap();

// unzip the tarball
let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap());
// extract the tar archive
let mut archive = tar::Archive::new(ungz_tar);
archive.unpack(&extract_dir).expect("Failed to extract!");
}
// crate is extracted, return a new Krate object which contains the path to the extracted
// sources that clippy can check
Crate {
version: self.version.clone(),
name: self.name.clone(),
path: extract_dir.join(format!("{}-{}/", self.name, self.version)),
match self {
CrateSource::CratesIo { name, version } => {
let extract_dir = PathBuf::from("target/lintcheck/crates");
let krate_download_dir = PathBuf::from("target/lintcheck/downloads");

// url to download the crate from crates.io
let url = format!("https://crates.io/api/v1/crates/{}/{}/download", name, version);
println!("Downloading and extracting {} {} from {}", name, version, url);
let _ = std::fs::create_dir("target/lintcheck/");
let _ = std::fs::create_dir(&krate_download_dir);
let _ = std::fs::create_dir(&extract_dir);

let krate_file_path = krate_download_dir.join(format!("{}-{}.crate.tar.gz", name, version));
// don't download/extract if we already have done so
if !krate_file_path.is_file() {
// create a file path to download and write the crate data into
let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap();
let mut krate_req = ureq::get(&url).call().unwrap().into_reader();
// copy the crate into the file
std::io::copy(&mut krate_req, &mut krate_dest).unwrap();

// unzip the tarball
let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap());
// extract the tar archive
let mut archive = tar::Archive::new(ungz_tar);
archive.unpack(&extract_dir).expect("Failed to extract!");
}
// crate is extracted, return a new Krate object which contains the path to the extracted
// sources that clippy can check
Crate {
version: version.clone(),
name: name.clone(),
path: extract_dir.join(format!("{}-{}/", name, version)),
}
},
CrateSource::Git { name, url, commit } => {
let repo_path = {
let mut repo_path = PathBuf::from("target/lintcheck/downloads");
// add a -git suffix in case we have the same crate from crates.io and a git repo
repo_path.push(format!("{}-git", name));
repo_path
};
// clone the repo if we have not done so
if !repo_path.is_dir() {
println!("Cloning {} and checking out {}", url, commit);
Command::new("git")
.arg("clone")
.arg(url)
.arg(&repo_path)
.output()
.expect("Failed to clone git repo!");
}
// check out the commit/branch/whatever
Command::new("git")
.arg("checkout")
.arg(commit)
.output()
.expect("Failed to check out commit");

Crate {
version: commit.clone(),
name: name.clone(),
path: repo_path,
}
},
}
}
}
Expand All @@ -114,7 +150,7 @@ impl Crate {

let shared_target_dir = clippy_project_root().join("target/lintcheck/shared_target_dir/");

let all_output = std::process::Command::new(cargo_clippy_path)
let all_output = std::process::Command::new(&cargo_clippy_path)
.env("CARGO_TARGET_DIR", shared_target_dir)
// lint warnings will look like this:
// src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter`
Expand All @@ -128,10 +164,16 @@ impl Crate {
])
.current_dir(&self.path)
.output()
.unwrap();
.unwrap_or_else(|error| {
panic!(
"Encountered error:\n{:?}\ncargo_clippy_path: {}\ncrate path:{}\n",
error,
&cargo_clippy_path.display(),
&self.path.display()
);
});
let stdout = String::from_utf8_lossy(&all_output.stdout);
let output_lines = stdout.lines();
//dbg!(&output_lines);
let warnings: Vec<ClippyWarning> = output_lines
.into_iter()
// get all clippy warnings
Expand Down Expand Up @@ -160,19 +202,40 @@ fn read_crates() -> Vec<CrateSource> {
let tomlcrates: Vec<TomlCrate> = crate_list
.crates
.into_iter()
.map(|(name, versions)| TomlCrate { name, versions })
.map(|(_cratename, tomlcrate)| tomlcrate)
.collect();

// flatten TomlCrates into CrateSources (one TomlCrates may represent several versions of a crate =>
// multiple Cratesources)
let mut crate_sources = Vec::new();
tomlcrates.into_iter().for_each(|tk| {
tk.versions.iter().for_each(|ver| {
crate_sources.push(CrateSource {
// if we have multiple versions, save each one
if let Some(ref versions) = tk.versions {
versions.iter().for_each(|ver| {
crate_sources.push(CrateSource::CratesIo {
name: tk.name.clone(),
version: ver.to_string(),
});
})
}
// otherwise, we should have a git source
if tk.git_url.is_some() && tk.git_hash.is_some() {
crate_sources.push(CrateSource::Git {
name: tk.name.clone(),
version: ver.to_string(),
url: tk.git_url.clone().unwrap(),
commit: tk.git_hash.clone().unwrap(),
});
})
}
// if we have a version as well as a git data OR only one git data, something is funky
if tk.versions.is_some() && (tk.git_url.is_some() || tk.git_hash.is_some())
|| tk.git_hash.is_some() != tk.git_url.is_some()
{
eprintln!("tomlkrate: {:?}", tk);
if tk.git_hash.is_some() != tk.git_url.is_some() {
panic!("Encountered TomlCrate with only one of git_hash and git_url!")
}
unreachable!("Failed to translate TomlCrate into CrateSource!");
}
});
crate_sources
}
Expand Down Expand Up @@ -228,8 +291,14 @@ pub fn run(clap_config: &ArgMatches) {
let crates = read_crates();

let clippy_warnings: Vec<ClippyWarning> = if let Some(only_one_crate) = clap_config.value_of("only") {
// if we don't have the specified crated in the .toml, throw an error
if !crates.iter().any(|krate| krate.name == only_one_crate) {
// if we don't have the specified crate in the .toml, throw an error
if !crates.iter().any(|krate| {
let name = match krate {
CrateSource::CratesIo { name, .. } => name,
CrateSource::Git { name, .. } => name,
};
name == only_one_crate
}) {
eprintln!(
"ERROR: could not find crate '{}' in clippy_dev/lintcheck_crates.toml",
only_one_crate
Expand Down
Loading