Skip to content

Work around LLVM issues with explicit register in inline asm #75014

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 17, 2020
Merged

Work around LLVM issues with explicit register in inline asm #75014

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 126 additions & 29 deletions src/librustc_codegen_llvm/asm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
for (idx, op) in operands.iter().enumerate() {
match *op {
InlineAsmOperandRef::Out { reg, late, place } => {
let ty = if let Some(place) = place {
let mut layout = None;
let ty = if let Some(ref place) = place {
layout = Some(&place.layout);
llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
} else {
// If the output is discarded, we don't really care what
Expand All @@ -141,20 +143,21 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
output_types.push(ty);
op_idx.insert(idx, constraints.len());
let prefix = if late { "=" } else { "=&" };
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout)));
}
InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
let ty = if let Some(ref out_place) = out_place {
llvm_fixup_output_type(self.cx, reg.reg_class(), &out_place.layout)
let layout = if let Some(ref out_place) = out_place {
&out_place.layout
} else {
// LLVM required tied operands to have the same type,
// so we just use the type of the input.
llvm_fixup_output_type(self.cx, reg.reg_class(), &in_value.layout)
&in_value.layout
};
let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout);
output_types.push(ty);
op_idx.insert(idx, constraints.len());
let prefix = if late { "=" } else { "=&" };
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
}
_ => {}
}
Expand All @@ -165,11 +168,11 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
for (idx, op) in operands.iter().enumerate() {
match *op {
InlineAsmOperandRef::In { reg, value } => {
let value =
let llval =
llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout);
inputs.push(value);
inputs.push(llval);
op_idx.insert(idx, constraints.len());
constraints.push(reg_to_llvm(reg));
constraints.push(reg_to_llvm(reg, Some(&value.layout)));
}
InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => {
let value = llvm_fixup_input(
Expand Down Expand Up @@ -410,10 +413,80 @@ fn inline_asm_call(
}
}

/// If the register is an xmm/ymm/zmm register then return its index.
fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
match reg {
InlineAsmReg::X86(reg)
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
Comment on lines +419 to +421
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
InlineAsmReg::X86(reg)
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
InlineAsmReg::X86(reg)
if RangeInclusive::new(X86InlineAsmReg::xmm0, X86InlineAsmReg::xmm15).contains(&reg) =>

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does RangeInclusive work on enums?

Copy link
Contributor

@tesuji tesuji Aug 1, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it is fieldless enum and implements PartialOrd and/or PartialEq I think.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can just write (X86InlineAsmReg::xmm0..=X86InlineAsmReg::xmm15).contains(&reg), too.

{
Some(reg as u32 - X86InlineAsmReg::xmm0 as u32)
}
InlineAsmReg::X86(reg)
if reg as u32 >= X86InlineAsmReg::ymm0 as u32
&& reg as u32 <= X86InlineAsmReg::ymm15 as u32 =>
{
Some(reg as u32 - X86InlineAsmReg::ymm0 as u32)
}
InlineAsmReg::X86(reg)
if reg as u32 >= X86InlineAsmReg::zmm0 as u32
&& reg as u32 <= X86InlineAsmReg::zmm31 as u32 =>
{
Some(reg as u32 - X86InlineAsmReg::zmm0 as u32)
}
_ => None,
}
}

/// If the register is an AArch64 vector register then return its index.
fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
match reg {
InlineAsmReg::AArch64(reg)
if reg as u32 >= AArch64InlineAsmReg::v0 as u32
&& reg as u32 <= AArch64InlineAsmReg::v31 as u32 =>
{
Some(reg as u32 - AArch64InlineAsmReg::v0 as u32)
}
_ => None,
}
}

/// Converts a register class to an LLVM constraint code.
fn reg_to_llvm(reg: InlineAsmRegOrRegClass) -> String {
fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>) -> String {
match reg {
InlineAsmRegOrRegClass::Reg(reg) => format!("{{{}}}", reg.name()),
// For vector registers LLVM wants the register name to match the type size.
InlineAsmRegOrRegClass::Reg(reg) => {
if let Some(idx) = xmm_reg_index(reg) {
let class = if let Some(layout) = layout {
match layout.size.bytes() {
64 => 'z',
32 => 'y',
_ => 'x',
}
} else {
// We use f32 as the type for discarded outputs
'x'
};
format!("{{{}mm{}}}", class, idx)
} else if let Some(idx) = a64_vreg_index(reg) {
let class = if let Some(layout) = layout {
match layout.size.bytes() {
16 => 'q',
8 => 'd',
4 => 's',
2 => 'h',
1 => 'd', // We fixup i8 to i8x8
_ => unreachable!(),
}
} else {
// We use i32 as the type for discarded outputs
's'
};
format!("{{{}{}}}", class, idx)
} else {
format!("{{{}}}", reg.name())
}
}
InlineAsmRegOrRegClass::RegClass(reg) => match reg {
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
Expand Down Expand Up @@ -600,18 +673,26 @@ fn llvm_fixup_input(
InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
Abi::Vector { .. },
) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
(
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
Abi::Scalar(s),
) => {
if let Primitive::Int(Integer::I32, _) = s.value {
bx.bitcast(value, bx.cx.type_f32())
} else {
value
}
}
(
InlineAsmRegClass::Arm(
ArmInlineAsmRegClass::sreg_low16
ArmInlineAsmRegClass::dreg
| ArmInlineAsmRegClass::dreg_low8
| ArmInlineAsmRegClass::qreg_low4
| ArmInlineAsmRegClass::dreg
| ArmInlineAsmRegClass::qreg,
| ArmInlineAsmRegClass::dreg_low16,
),
Abi::Scalar(s),
) => {
if let Primitive::Int(Integer::I32, _) = s.value {
bx.bitcast(value, bx.cx.type_f32())
if let Primitive::Int(Integer::I64, _) = s.value {
bx.bitcast(value, bx.cx.type_f64())
} else {
value
}
Expand Down Expand Up @@ -660,18 +741,26 @@ fn llvm_fixup_output(
InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
Abi::Vector { .. },
) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
(
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
Abi::Scalar(s),
) => {
if let Primitive::Int(Integer::I32, _) = s.value {
bx.bitcast(value, bx.cx.type_i32())
} else {
value
}
}
(
InlineAsmRegClass::Arm(
ArmInlineAsmRegClass::sreg_low16
ArmInlineAsmRegClass::dreg
| ArmInlineAsmRegClass::dreg_low8
| ArmInlineAsmRegClass::qreg_low4
| ArmInlineAsmRegClass::dreg
| ArmInlineAsmRegClass::qreg,
| ArmInlineAsmRegClass::dreg_low16,
),
Abi::Scalar(s),
) => {
if let Primitive::Int(Integer::I32, _) = s.value {
bx.bitcast(value, bx.cx.type_i32())
if let Primitive::Int(Integer::I64, _) = s.value {
bx.bitcast(value, bx.cx.type_i64())
} else {
value
}
Expand Down Expand Up @@ -715,18 +804,26 @@ fn llvm_fixup_output_type(
InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
Abi::Vector { .. },
) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
(
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
Abi::Scalar(s),
) => {
if let Primitive::Int(Integer::I32, _) = s.value {
cx.type_f32()
} else {
layout.llvm_type(cx)
}
}
(
InlineAsmRegClass::Arm(
ArmInlineAsmRegClass::sreg_low16
ArmInlineAsmRegClass::dreg
| ArmInlineAsmRegClass::dreg_low8
| ArmInlineAsmRegClass::qreg_low4
| ArmInlineAsmRegClass::dreg
| ArmInlineAsmRegClass::qreg,
| ArmInlineAsmRegClass::dreg_low16,
),
Abi::Scalar(s),
) => {
if let Primitive::Int(Integer::I32, _) = s.value {
cx.type_f32()
if let Primitive::Int(Integer::I64, _) = s.value {
cx.type_f64()
} else {
layout.llvm_type(cx)
}
Expand Down
Loading