bootstrap/compiletest: implement "crashes" tests that fail if no ice is reproduced

Author: matthiaskrgr

src/bootstrap/src/core/build_steps/test.rs

@@ -1400,6 +1400,8 @@ impl Step for RunMakeSupport {
 
 default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" });
 
+default_test!(Crashes { path: "tests/crashes", mode: "crashes", suite: "crashes" });
+
 default_test!(RunPassValgrind {
     path: "tests/run-pass-valgrind",
     mode: "run-pass-valgrind",

src/bootstrap/src/core/builder.rs

@@ -745,6 +745,7 @@ impl<'a> Builder<'a> {
                 test::ExpandYamlAnchors,
                 test::Tidy,
                 test::Ui,
+                test::Crashes,
                 test::RunPassValgrind,
                 test::Coverage,
                 test::CoverageMap,

src/tools/compiletest/src/common.rs

@@ -69,6 +69,7 @@ string_enum! {
         Assembly => "assembly",
         CoverageMap => "coverage-map",
         CoverageRun => "coverage-run",
+        Crashes => "crashes",
     }
 }
 

src/tools/compiletest/src/header.rs

@@ -625,6 +625,7 @@ impl TestProps {
     fn update_pass_mode(&mut self, ln: &str, revision: Option<&str>, config: &Config) {
         let check_no_run = |s| match (config.mode, s) {
             (Mode::Ui, _) => (),
+            (Mode::Crashes, "should-ice") => (),
             (Mode::Codegen, "build-pass") => (),
             (Mode::Incremental, _) => {
                 if revision.is_some() && !self.revisions.iter().all(|r| r.starts_with("cfail")) {

src/tools/compiletest/src/lib.rs

@@ -65,7 +65,8 @@ pub fn parse_config(args: Vec<String>) -> Config {
             "mode",
             "which sort of compile tests to run",
             "run-pass-valgrind | pretty | debug-info | codegen | rustdoc \
-            | rustdoc-json | codegen-units | incremental | run-make | ui | js-doc-test | mir-opt | assembly",
+            | rustdoc-json | codegen-units | incremental | run-make | ui \
+            | js-doc-test | mir-opt | assembly | crashes",
         )
         .reqopt(
             "",
@@ -82,7 +83,12 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .optopt("", "run", "whether to execute run-* tests", "auto | always | never")
         .optflag("", "ignored", "run tests marked as ignored")
         .optflag("", "with-debug-assertions", "whether to run tests with `ignore-debug` header")
-        .optmulti("", "skip", "skip tests matching SUBSTRING. Can be passed multiple times", "SUBSTRING")
+        .optmulti(
+            "",
+            "skip",
+            "skip tests matching SUBSTRING. Can be passed multiple times",
+            "SUBSTRING",
+        )
         .optflag("", "exact", "filters match exactly")
         .optopt(
             "",
@@ -145,7 +151,11 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .optflag("", "profiler-support", "is the profiler runtime enabled for this target")
         .optflag("h", "help", "show this message")
         .reqopt("", "channel", "current Rust channel", "CHANNEL")
-        .optflag("", "git-hash", "run tests which rely on commit version being compiled into the binaries")
+        .optflag(
+            "",
+            "git-hash",
+            "run tests which rely on commit version being compiled into the binaries",
+        )
         .optopt("", "edition", "default Rust edition", "EDITION")
         .reqopt("", "git-repository", "name of the git repository", "ORG/REPO")
         .reqopt("", "nightly-branch", "name of the git branch for nightly", "BRANCH");

src/tools/compiletest/src/runtest.rs

@@ -4,7 +4,7 @@ use crate::common::{
     expected_output_path, UI_EXTENSIONS, UI_FIXED, UI_STDERR, UI_STDOUT, UI_SVG, UI_WINDOWS_SVG,
 };
 use crate::common::{incremental_dir, output_base_dir, output_base_name, output_testname_unique};
-use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJson, Ui};
+use crate::common::{Assembly, Crashes, Incremental, JsDocTest, MirOpt, RunMake, RustdocJson, Ui};
 use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
 use crate::common::{CompareMode, FailMode, PassMode};
 use crate::common::{Config, TestPaths};
@@ -244,7 +244,7 @@ impl<'test> TestCx<'test> {
     /// Code executed for each revision in turn (or, if there are no
     /// revisions, exactly once, with revision == None).
     fn run_revision(&self) {
-        if self.props.should_ice && self.config.mode != Incremental {
+        if self.props.should_ice && self.config.mode != Incremental && self.config.mode != Crashes {
             self.fatal("cannot use should-ice in a test that is not cfail");
         }
         match self.config.mode {
@@ -263,6 +263,7 @@ impl<'test> TestCx<'test> {
             JsDocTest => self.run_js_doc_test(),
             CoverageMap => self.run_coverage_map_test(),
             CoverageRun => self.run_coverage_run_test(),
+            Crashes => self.run_crash_test(),
         }
     }
 
@@ -295,6 +296,7 @@ impl<'test> TestCx<'test> {
         match self.config.mode {
             JsDocTest => true,
             Ui => pm.is_some() || self.props.fail_mode > Some(FailMode::Build),
+            Crashes => false,
             Incremental => {
                 let revision =
                     self.revision.expect("incremental tests require a list of revisions");
@@ -359,6 +361,17 @@ impl<'test> TestCx<'test> {
         self.check_forbid_output(&output_to_check, &proc_res);
     }
 
+    fn run_crash_test(&self) {
+        let pm = self.pass_mode();
+        let proc_res = self.compile_test(WillExecute::No, self.should_emit_metadata(pm));
+
+        // if a test does not crash, consider it an error
+        match proc_res.status.code() {
+            Some(101) => (),
+            _ => self.fatal("expected ICE"),
+        }
+    }
+
     fn run_rfail_test(&self) {
         let pm = self.pass_mode();
         let should_run = self.run_if_enabled();
@@ -2517,7 +2530,7 @@ impl<'test> TestCx<'test> {
                 rustc.arg("-Cdebug-assertions=no");
             }
             RunPassValgrind | Pretty | DebugInfo | Rustdoc | RustdocJson | RunMake
-            | CodegenUnits | JsDocTest => {
+            | CodegenUnits | JsDocTest | Crashes => {
                 // do not use JSON output
             }
         }

src/tools/opt-dist/src/tests.rs

@@ -96,6 +96,7 @@ llvm-config = "{llvm_config}"
         "tests/pretty",
         "tests/run-pass-valgrind",
         "tests/ui",
+        "tests/crases",
     ];
     for test_path in env.skipped_tests() {
         args.extend(["--skip", test_path]);

tests/crashes/no-ice.rs

@@ -0,0 +1 @@
+pub fn main() {}