Add additional regression tests for PGO

src/test/run-make-fulldeps/pgo-branch-weights/Makefile

@@ -0,0 +1,35 @@
+# needs-profiler-support
+
+-include ../tools.mk
+
+# This test makes sure that instrumented binaries record the right counts for
+# functions being called and branches being taken. We run an instrumented binary
+# with an argument that causes a know path through the program and then check
+# that the expected counts get added to the use-phase LLVM IR.
+
+# LLVM doesn't support instrumenting binaries that use SEH:
+# https://github.com/rust-lang/rust/issues/61002
+#
+# Things work fine with -Cpanic=abort though.
+ifdef IS_MSVC
+COMMON_FLAGS=-Cpanic=abort
+endif
+
+all:
+	# We don't compile `opaque` with either optimizations or instrumentation.
+	# We don't compile `opaque` with either optimizations or instrumentation.
+	$(RUSTC) $(COMMON_FLAGS) opaque.rs
+	# Compile the test program with instrumentation
+	mkdir -p "$(TMPDIR)"/prof_data_dir
+	$(RUSTC) $(COMMON_FLAGS) interesting.rs \
+		-Cprofile-generate="$(TMPDIR)"/prof_data_dir -O -Ccodegen-units=1
+	$(RUSTC) $(COMMON_FLAGS) main.rs -Cprofile-generate="$(TMPDIR)"/prof_data_dir -O
+	# The argument below generates to the expected branch weights
+	$(call RUN,main aaaaaaaaaaaa2bbbbbbbbbbbb2bbbbbbbbbbbbbbbbcc) || exit 1
+	"$(LLVM_BIN_DIR)"/llvm-profdata merge \
+		-o "$(TMPDIR)"/prof_data_dir/merged.profdata \
+		"$(TMPDIR)"/prof_data_dir
+	$(RUSTC) $(COMMON_FLAGS) interesting.rs \
+		-Cprofile-use="$(TMPDIR)"/prof_data_dir/merged.profdata -O \
+		-Ccodegen-units=1 --emit=llvm-ir
+	cat "$(TMPDIR)"/interesting.ll | "$(LLVM_FILECHECK)" filecheck-patterns.txt

src/test/run-make-fulldeps/pgo-branch-weights/filecheck-patterns.txt

@@ -0,0 +1,24 @@
+
+# First, establish that certain !prof labels are attached to the expected
+# functions and branching instructions.
+
+CHECK: define void @function_called_twice(i32 %c) {{.*}} !prof [[function_called_twice_id:![0-9]+]] {
+CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !prof [[branch_weights0:![0-9]+]]
+
+CHECK: define void @function_called_42_times(i32 %c) {{.*}} !prof [[function_called_42_times_id:![0-9]+]] {
+CHECK:      switch i32 %c, label {{.*}} [
+CHECK-NEXT:     i32 97, label {{.*}}
+CHECK-NEXT:     i32 98, label {{.*}}
+CHECK-NEXT: ], !prof [[branch_weights1:![0-9]+]]
+
+CHECK: define void @function_called_never(i32 {{.*}} !prof [[function_called_never_id:![0-9]+]] {
+
+
+
+# Now check that those !prof tags hold the expected counts
+
+CHECK: [[function_called_twice_id]] = !{!"function_entry_count", i64 2}
+CHECK: [[branch_weights0]] = !{!"branch_weights", i32 2, i32 0}
+CHECK: [[function_called_42_times_id]] = !{!"function_entry_count", i64 42}
+CHECK: [[branch_weights1]] = !{!"branch_weights", i32 2, i32 12, i32 28}
+CHECK: [[function_called_never_id]] = !{!"function_entry_count", i64 0}

src/test/run-make-fulldeps/pgo-branch-weights/interesting.rs

@@ -0,0 +1,40 @@
+#![crate_name="interesting"]
+#![crate_type="rlib"]
+
+extern crate opaque;
+
+#[no_mangle]
+#[inline(never)]
+pub fn function_called_twice(c: char) {
+    if c == '2' {
+        // This branch is taken twice
+        opaque::f1();
+    } else {
+        // This branch is never taken
+        opaque::f2();
+    }
+}
+
+#[no_mangle]
+#[inline(never)]
+pub fn function_called_42_times(c: char) {
+    if c == 'a' {
+        // This branch is taken 12 times
+        opaque::f1();
+    } else {
+
+        if c == 'b' {
+            // This branch is taken 28 times
+            opaque::f2();
+        } else {
+            // This branch is taken 2 times
+            opaque::f3();
+        }
+    }
+}
+
+#[no_mangle]
+#[inline(never)]
+pub fn function_called_never(_: char) {
+    opaque::f1();
+}

src/test/run-make-fulldeps/pgo-branch-weights/main.rs

@@ -0,0 +1,17 @@
+extern crate interesting;
+
+fn main() {
+    let arg = std::env::args().skip(1).next().unwrap();
+
+    for c in arg.chars() {
+        if c == '2' {
+            interesting::function_called_twice(c);
+        } else {
+            interesting::function_called_42_times(c);
+        }
+
+        if c == '0' {
+            interesting::function_called_never(c);
+        }
+    }
+}

src/test/run-make-fulldeps/pgo-branch-weights/opaque.rs

@@ -0,0 +1,6 @@
+#![crate_name="opaque"]
+#![crate_type="rlib"]
+
+pub fn f1() {}
+pub fn f2() {}
+pub fn f3() {}

src/test/run-make-fulldeps/pgo-indirect-call-promotion/Makefile

@@ -0,0 +1,36 @@
+# needs-profiler-support
+
+-include ../tools.mk
+
+# This test makes sure that indirect call promotion is performed. The test
+# programs calls the same function a thousand times through a function pointer.
+# Only PGO data provides the information that it actually always is the same
+# function. We verify that the indirect call promotion pass inserts a check
+# whether it can make a direct call instead of the indirect call.
+
+# LLVM doesn't support instrumenting binaries that use SEH:
+# https://github.com/rust-lang/rust/issues/61002
+#
+# Things work fine with -Cpanic=abort though.
+ifdef IS_MSVC
+COMMON_FLAGS=-Cpanic=abort
+endif
+
+all:
+	# We don't compile `opaque` with either optimizations or instrumentation.
+	# We don't compile `opaque` with either optimizations or instrumentation.
+	$(RUSTC) $(COMMON_FLAGS) opaque.rs
+	# Compile the test program with instrumentation
+	mkdir -p "$(TMPDIR)"/prof_data_dir
+	$(RUSTC) $(COMMON_FLAGS) interesting.rs \
+		-Cprofile-generate="$(TMPDIR)"/prof_data_dir -O -Ccodegen-units=1
+	$(RUSTC) $(COMMON_FLAGS) main.rs -Cprofile-generate="$(TMPDIR)"/prof_data_dir -O
+	# The argument below generates to the expected branch weights
+	$(call RUN,main) || exit 1
+	"$(LLVM_BIN_DIR)"/llvm-profdata merge \
+		-o "$(TMPDIR)"/prof_data_dir/merged.profdata \
+		"$(TMPDIR)"/prof_data_dir
+	$(RUSTC) $(COMMON_FLAGS) interesting.rs \
+		-Cprofile-use="$(TMPDIR)"/prof_data_dir/merged.profdata -O \
+		-Ccodegen-units=1 --emit=llvm-ir
+	cat "$(TMPDIR)"/interesting.ll | "$(LLVM_FILECHECK)" filecheck-patterns.txt

src/test/run-make-fulldeps/pgo-indirect-call-promotion/filecheck-patterns.txt

@@ -0,0 +1,16 @@
+CHECK: define void @call_a_bunch_of_functions({{.*}} {
+
+# Make sure that indirect call promotion inserted a check against the most
+# frequently called function.
+CHECK: %{{.*}} = icmp eq void ()* %{{.*}}, @function_called_always
+
+# Check that the call to `function_called_always` was inlined, so that we
+# directly call `opaque_f1` from the upstream crate.
+CHECK: call void @opaque_f1()
+
+
+# Same checks as above, repeated for the trait object case
+
+CHECK: define void @call_a_bunch_of_trait_methods({{.*}}
+CHECK: %{{.*}} = icmp eq void ({}*)* %{{.*}}, {{.*}} @foo
+CHECK: tail call void @opaque_f2()

src/test/run-make-fulldeps/pgo-indirect-call-promotion/interesting.rs

@@ -0,0 +1,56 @@
+#![crate_name="interesting"]
+#![crate_type="rlib"]
+
+extern crate opaque;
+
+#[no_mangle]
+pub fn function_called_always() {
+    opaque::opaque_f1();
+}
+
+#[no_mangle]
+pub fn function_called_never() {
+    opaque::opaque_f2();
+}
+
+#[no_mangle]
+pub fn call_a_bunch_of_functions(fns: &[fn()]) {
+
+    // Indirect call promotion transforms the below into something like
+    //
+    // for f in fns {
+    //     if f == function_called_always {
+    //         function_called_always()
+    //     } else {
+    //         f();
+    //     }
+    // }
+    //
+    // where `function_called_always` actually gets inlined too.
+
+    for f in fns {
+        f();
+    }
+}
+
+
+pub trait Foo {
+    fn foo(&self);
+}
+
+impl Foo for u32 {
+
+    #[no_mangle]
+    fn foo(&self) {
+        opaque::opaque_f2();
+    }
+}
+
+#[no_mangle]
+pub fn call_a_bunch_of_trait_methods(trait_objects: &[&dyn Foo]) {
+
+    // Same as above, just with vtables in between
+    for x in trait_objects {
+        x.foo();
+    }
+}

src/test/run-make-fulldeps/pgo-indirect-call-promotion/main.rs

@@ -0,0 +1,14 @@
+extern crate interesting;
+
+fn main() {
+    // function pointer case
+    let fns: Vec<_> = std::iter::repeat(interesting::function_called_always as fn())
+        .take(1000)
+        .collect();
+    interesting::call_a_bunch_of_functions(&fns[..]);
+
+    // Trait object case
+    let trait_objects = vec![0u32; 1000];
+    let trait_objects: Vec<_> = trait_objects.iter().map(|x| x as &dyn interesting::Foo).collect();
+    interesting::call_a_bunch_of_trait_methods(&trait_objects[..]);
+}

src/test/run-make-fulldeps/pgo-indirect-call-promotion/opaque.rs

@@ -0,0 +1,7 @@
+#![crate_name="opaque"]
+#![crate_type="rlib"]
+
+#[no_mangle]
+pub fn opaque_f1() {}
+#[no_mangle]
+pub fn opaque_f2() {}