Use unions for uninhabitedness checking rather than mem::transmute

Author: varkor

src/librustc/ty/layout.rs

@@ -561,7 +561,7 @@ impl<'a, 'tcx> LayoutCx<'tcx, TyCtxt<'a, 'tcx, 'tcx>> {
                 let size = element.size.checked_mul(count, dl)
                     .ok_or(LayoutError::SizeOverflow(ty))?;
 
-                let abi = if size != Size::ZERO && ty.conservative_is_uninhabited(tcx) {
+                let abi = if count != 0 && ty.conservative_is_uninhabited(tcx) {
                     Abi::Uninhabited
                 } else {
                     Abi::Aggregate { sized: true }

src/test/ui/consts/const-eval/ub-uninhabit.rs

@@ -1,28 +1,22 @@
-// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
 #![feature(const_transmute)]
 
 use std::mem;
 
 #[derive(Copy, Clone)]
 enum Bar {}
 
-const BAD_BAD_BAD: Bar = unsafe { mem::transmute(()) };
-//~^ ERROR this constant cannot be used
+union TransmuteUnion<A: Clone + Copy, B: Clone + Copy> {
+    a: A,
+    b: B,
+}
+
+const BAD_BAD_BAD: Bar = unsafe { (TransmuteUnion::<(), Bar> { a: () }).b };
+//~^ ERROR this constant likely exhibits undefined behavior
 
 const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
 //~^ ERROR this constant likely exhibits undefined behavior
 
-const BAD_BAD_ARRAY: [Bar; 1] = unsafe { mem::transmute(()) };
-//~^ ERROR this constant cannot be used
+const BAD_BAD_ARRAY: [Bar; 1] = unsafe { (TransmuteUnion::<(), [Bar; 1]> { a: () }).b };
+//~^ ERROR this constant likely exhibits undefined behavior
 
-fn main() {
-}
+fn main() {}

src/test/ui/consts/const-eval/ub-uninhabit.stderr

@@ -1,28 +1,26 @@
-error: this constant cannot be used
-  --> $DIR/ub-uninhabit.rs:18:1
+error[E0080]: this constant likely exhibits undefined behavior
+  --> $DIR/ub-uninhabit.rs:13:1
    |
-LL | const BAD_BAD_BAD: Bar = unsafe { mem::transmute(()) };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^------------------^^^
-   |                                   |
-   |                                   entered unreachable code
+LL | const BAD_BAD_BAD: Bar = unsafe { (TransmuteUnion::<(), Bar> { a: () }).b };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a value of an uninhabited type
    |
-   = note: #[deny(const_err)] on by default
+   = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rust compiler repository if you believe it should not be considered undefined behavior
 
 error[E0080]: this constant likely exhibits undefined behavior
-  --> $DIR/ub-uninhabit.rs:21:1
+  --> $DIR/ub-uninhabit.rs:16:1
    |
 LL | const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a value of an uninhabited type at .<deref>
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rust compiler repository if you believe it should not be considered undefined behavior
 
-error: this constant cannot be used
-  --> $DIR/ub-uninhabit.rs:24:1
+error[E0080]: this constant likely exhibits undefined behavior
+  --> $DIR/ub-uninhabit.rs:19:1
    |
-LL | const BAD_BAD_ARRAY: [Bar; 1] = unsafe { mem::transmute(()) };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^------------------^^^
-   |                                          |
-   |                                          entered unreachable code
+LL | const BAD_BAD_ARRAY: [Bar; 1] = unsafe { (TransmuteUnion::<(), [Bar; 1]> { a: () }).b };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a value of an uninhabited type
+   |
+   = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rust compiler repository if you believe it should not be considered undefined behavior
 
 error: aborting due to 3 previous errors