Skip to content

[ExpandMemCmp] Optimize ExpandMemCmp to reduce instruction count on x86 #69609

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 17 additions & 16 deletions llvm/lib/CodeGen/ExpandMemCmp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -307,19 +307,20 @@ MemCmpExpansion::LoadPair MemCmpExpansion::getLoadPair(Type *LoadSizeType,
if (!Rhs)
Rhs = Builder.CreateAlignedLoad(LoadSizeType, RhsSource, RhsAlign);

// Swap bytes if required.
if (NeedsBSwap) {
Function *Bswap = Intrinsic::getDeclaration(CI->getModule(),
Intrinsic::bswap, LoadSizeType);
Lhs = Builder.CreateCall(Bswap, Lhs);
Rhs = Builder.CreateCall(Bswap, Rhs);
}

// Zero extend if required.
if (CmpSizeType != nullptr && CmpSizeType != LoadSizeType) {
Lhs = Builder.CreateZExt(Lhs, CmpSizeType);
Rhs = Builder.CreateZExt(Rhs, CmpSizeType);
}

// Swap bytes if required.
if (NeedsBSwap) {
Type *BSwapType = CmpSizeType ? CmpSizeType : LoadSizeType;
Function *Bswap =
Intrinsic::getDeclaration(CI->getModule(), Intrinsic::bswap, BSwapType);
Lhs = Builder.CreateCall(Bswap, Lhs);
Rhs = Builder.CreateCall(Bswap, Rhs);
}
return {Lhs, Rhs};
}

Expand Down Expand Up @@ -694,10 +695,10 @@ Value *MemCmpExpansion::getMemCmpExpansion() {
/// %17 = getelementptr i32, i32* %15, i32 2
/// %18 = load i32, i32* %16
/// %19 = load i32, i32* %17
/// %20 = call i32 @llvm.bswap.i32(i32 %18)
/// %21 = call i32 @llvm.bswap.i32(i32 %19)
/// %22 = zext i32 %20 to i64
/// %23 = zext i32 %21 to i64
/// %20 = zext i32 %18 to i64
/// %21 = zext i32 %19 to i64
/// %22 = call i64 @llvm.bswap.i64(i64 %20)
/// %23 = call i64 @llvm.bswap.i64(i64 %21)
/// %24 = sub i64 %22, %23
/// %25 = icmp ne i64 %24, 0
/// br i1 %25, label %res_block, label %loadbb2
Expand All @@ -710,10 +711,10 @@ Value *MemCmpExpansion::getMemCmpExpansion() {
/// %31 = getelementptr i16, i16* %29, i16 6
/// %32 = load i16, i16* %30
/// %33 = load i16, i16* %31
/// %34 = call i16 @llvm.bswap.i16(i16 %32)
/// %35 = call i16 @llvm.bswap.i16(i16 %33)
/// %36 = zext i16 %34 to i64
/// %37 = zext i16 %35 to i64
/// %34 = zext i16 %32 to i64
/// %35 = zext i16 %33 to i64
/// %36 = call i64 @llvm.bswap.i64(i16 %34)
/// %37 = call i64 @llvm.bswap.i64(i16 %35)
/// %38 = sub i64 %36, %37
/// %39 = icmp ne i64 %38, 0
/// br i1 %39, label %res_block, label %loadbb3
Expand Down
26 changes: 10 additions & 16 deletions llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,12 @@ define i1 @length0_lt(ptr %X, ptr %Y) nounwind {
define i32 @length2(ptr %X, ptr %Y) nounwind {
; X86-LABEL: length2:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: movzwl (%ecx), %ecx
; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: rolw $8, %cx
; X86-NEXT: rolw $8, %dx
; X86-NEXT: movzwl %cx, %eax
; X86-NEXT: movzwl %dx, %ecx
; X86-NEXT: bswapl %eax
; X86-NEXT: bswapl %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: retl
%m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 2) nounwind
Expand All @@ -75,14 +73,12 @@ define i1 @length2_eq(ptr %X, ptr %Y) nounwind {
define i1 @length2_lt(ptr %X, ptr %Y) nounwind {
; X86-LABEL: length2_lt:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: movzwl (%ecx), %ecx
; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: rolw $8, %cx
; X86-NEXT: rolw $8, %dx
; X86-NEXT: movzwl %cx, %eax
; X86-NEXT: movzwl %dx, %ecx
; X86-NEXT: bswapl %eax
; X86-NEXT: bswapl %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: shrl $31, %eax
; X86-NEXT: # kill: def $al killed $al killed $eax
Expand All @@ -99,10 +95,8 @@ define i1 @length2_gt(ptr %X, ptr %Y) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movzwl (%ecx), %ecx
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: rolw $8, %cx
; X86-NEXT: rolw $8, %ax
; X86-NEXT: movzwl %cx, %ecx
; X86-NEXT: movzwl %ax, %eax
; X86-NEXT: bswapl %ecx
; X86-NEXT: bswapl %eax
; X86-NEXT: subl %eax, %ecx
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: setg %al
Expand Down
22 changes: 8 additions & 14 deletions llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,8 @@ define i32 @length2(ptr %X, ptr %Y) nounwind {
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: movzwl (%rsi), %ecx
; X64-NEXT: rolw $8, %ax
; X64-NEXT: rolw $8, %cx
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: movzwl %cx, %ecx
; X64-NEXT: bswapl %eax
; X64-NEXT: bswapl %ecx
; X64-NEXT: subl %ecx, %eax
; X64-NEXT: retq
%m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 2) nounwind
Expand All @@ -79,10 +77,8 @@ define i1 @length2_lt(ptr %X, ptr %Y) nounwind {
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: movzwl (%rsi), %ecx
; X64-NEXT: rolw $8, %ax
; X64-NEXT: rolw $8, %cx
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: movzwl %cx, %ecx
; X64-NEXT: bswapl %eax
; X64-NEXT: bswapl %ecx
; X64-NEXT: subl %ecx, %eax
; X64-NEXT: shrl $31, %eax
; X64-NEXT: # kill: def $al killed $al killed $eax
Expand All @@ -97,10 +93,8 @@ define i1 @length2_gt(ptr %X, ptr %Y) nounwind {
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: movzwl (%rsi), %ecx
; X64-NEXT: rolw $8, %ax
; X64-NEXT: rolw $8, %cx
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: movzwl %cx, %ecx
; X64-NEXT: bswapl %eax
; X64-NEXT: bswapl %ecx
; X64-NEXT: subl %ecx, %eax
; X64-NEXT: testl %eax, %eax
; X64-NEXT: setg %al
Expand Down Expand Up @@ -511,8 +505,8 @@ define i32 @length12(ptr %X, ptr %Y) nounwind {
; X64-NEXT: # %bb.1: # %loadbb1
; X64-NEXT: movl 8(%rdi), %ecx
; X64-NEXT: movl 8(%rsi), %edx
; X64-NEXT: bswapl %ecx
; X64-NEXT: bswapl %edx
; X64-NEXT: bswapq %rcx
; X64-NEXT: bswapq %rdx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: je .LBB29_3
Expand Down
10 changes: 4 additions & 6 deletions llvm/test/CodeGen/X86/memcmp-optsize-x32.ll
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,12 @@ declare dso_local i32 @bcmp(ptr, ptr, i32)
define i32 @length2(ptr %X, ptr %Y) nounwind optsize {
; X86-LABEL: length2:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: movzwl (%ecx), %ecx
; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: rolw $8, %cx
; X86-NEXT: rolw $8, %dx
; X86-NEXT: movzwl %cx, %eax
; X86-NEXT: movzwl %dx, %ecx
; X86-NEXT: bswapl %eax
; X86-NEXT: bswapl %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: retl
%m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 2) nounwind
Expand Down
10 changes: 4 additions & 6 deletions llvm/test/CodeGen/X86/memcmp-optsize.ll
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@ define i32 @length2(ptr %X, ptr %Y) nounwind optsize {
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: movzwl (%rsi), %ecx
; X64-NEXT: rolw $8, %ax
; X64-NEXT: rolw $8, %cx
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: movzwl %cx, %ecx
; X64-NEXT: bswapl %eax
; X64-NEXT: bswapl %ecx
; X64-NEXT: subl %ecx, %eax
; X64-NEXT: retq
%m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 2) nounwind
Expand Down Expand Up @@ -251,8 +249,8 @@ define i32 @length12(ptr %X, ptr %Y) nounwind optsize {
; X64-NEXT: # %bb.1: # %loadbb1
; X64-NEXT: movl 8(%rdi), %ecx
; X64-NEXT: movl 8(%rsi), %edx
; X64-NEXT: bswapl %ecx
; X64-NEXT: bswapl %edx
; X64-NEXT: bswapq %rcx
; X64-NEXT: bswapq %rdx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: je .LBB15_3
Expand Down
10 changes: 4 additions & 6 deletions llvm/test/CodeGen/X86/memcmp-pgso-x32.ll
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,12 @@ declare dso_local i32 @bcmp(ptr, ptr, i32)
define i32 @length2(ptr %X, ptr %Y) nounwind !prof !14 {
; X86-LABEL: length2:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: movzwl (%ecx), %ecx
; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: rolw $8, %cx
; X86-NEXT: rolw $8, %dx
; X86-NEXT: movzwl %cx, %eax
; X86-NEXT: movzwl %dx, %ecx
; X86-NEXT: bswapl %eax
; X86-NEXT: bswapl %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: retl
%m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 2) nounwind
Expand Down
10 changes: 4 additions & 6 deletions llvm/test/CodeGen/X86/memcmp-pgso.ll
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@ define i32 @length2(ptr %X, ptr %Y) nounwind !prof !14 {
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: movzwl (%rsi), %ecx
; X64-NEXT: rolw $8, %ax
; X64-NEXT: rolw $8, %cx
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: movzwl %cx, %ecx
; X64-NEXT: bswapl %eax
; X64-NEXT: bswapl %ecx
; X64-NEXT: subl %ecx, %eax
; X64-NEXT: retq
%m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 2) nounwind
Expand Down Expand Up @@ -251,8 +249,8 @@ define i32 @length12(ptr %X, ptr %Y) nounwind !prof !14 {
; X64-NEXT: # %bb.1: # %loadbb1
; X64-NEXT: movl 8(%rdi), %ecx
; X64-NEXT: movl 8(%rsi), %edx
; X64-NEXT: bswapl %ecx
; X64-NEXT: bswapl %edx
; X64-NEXT: bswapq %rcx
; X64-NEXT: bswapq %rdx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: je .LBB15_3
Expand Down
36 changes: 14 additions & 22 deletions llvm/test/CodeGen/X86/memcmp-x32.ll
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,12 @@ define i1 @length0_lt(ptr %X, ptr %Y) nounwind {
define i32 @length2(ptr %X, ptr %Y) nounwind {
; X86-LABEL: length2:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: movzwl (%ecx), %ecx
; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: rolw $8, %cx
; X86-NEXT: rolw $8, %dx
; X86-NEXT: movzwl %cx, %eax
; X86-NEXT: movzwl %dx, %ecx
; X86-NEXT: bswapl %eax
; X86-NEXT: bswapl %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: retl
%m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 2) nounwind
Expand All @@ -62,9 +60,8 @@ define i32 @length2_const(ptr %X, ptr %Y) nounwind {
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: rolw $8, %ax
; X86-NEXT: movzwl %ax, %eax
; X86-NEXT: addl $-12594, %eax # imm = 0xCECE
; X86-NEXT: bswapl %eax
; X86-NEXT: addl $-825360384, %eax # imm = 0xCECE0000
; X86-NEXT: retl
%m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([513 x i8], ptr @.str, i32 0, i32 1), i32 2) nounwind
ret i32 %m
Expand All @@ -75,9 +72,8 @@ define i1 @length2_gt_const(ptr %X, ptr %Y) nounwind {
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: rolw $8, %ax
; X86-NEXT: movzwl %ax, %eax
; X86-NEXT: addl $-12594, %eax # imm = 0xCECE
; X86-NEXT: bswapl %eax
; X86-NEXT: addl $-825360384, %eax # imm = 0xCECE0000
; X86-NEXT: testl %eax, %eax
; X86-NEXT: setg %al
; X86-NEXT: retl
Expand All @@ -103,14 +99,12 @@ define i1 @length2_eq(ptr %X, ptr %Y) nounwind {
define i1 @length2_lt(ptr %X, ptr %Y) nounwind {
; X86-LABEL: length2_lt:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: movzwl (%ecx), %ecx
; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: rolw $8, %cx
; X86-NEXT: rolw $8, %dx
; X86-NEXT: movzwl %cx, %eax
; X86-NEXT: movzwl %dx, %ecx
; X86-NEXT: bswapl %eax
; X86-NEXT: bswapl %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: shrl $31, %eax
; X86-NEXT: # kill: def $al killed $al killed $eax
Expand All @@ -127,10 +121,8 @@ define i1 @length2_gt(ptr %X, ptr %Y) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movzwl (%ecx), %ecx
; X86-NEXT: movzwl (%eax), %eax
; X86-NEXT: rolw $8, %cx
; X86-NEXT: rolw $8, %ax
; X86-NEXT: movzwl %cx, %ecx
; X86-NEXT: movzwl %ax, %eax
; X86-NEXT: bswapl %ecx
; X86-NEXT: bswapl %eax
; X86-NEXT: subl %eax, %ecx
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: setg %al
Expand Down
32 changes: 12 additions & 20 deletions llvm/test/CodeGen/X86/memcmp.ll
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,8 @@ define i32 @length2(ptr %X, ptr %Y) nounwind {
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: movzwl (%rsi), %ecx
; X64-NEXT: rolw $8, %ax
; X64-NEXT: rolw $8, %cx
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: movzwl %cx, %ecx
; X64-NEXT: bswapl %eax
; X64-NEXT: bswapl %ecx
; X64-NEXT: subl %ecx, %eax
; X64-NEXT: retq
%m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 2) nounwind
Expand All @@ -65,9 +63,8 @@ define i32 @length2_const(ptr %X, ptr %Y) nounwind {
; X64-LABEL: length2_const:
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: rolw $8, %ax
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: addl $-12594, %eax # imm = 0xCECE
; X64-NEXT: bswapl %eax
; X64-NEXT: addl $-825360384, %eax # imm = 0xCECE0000
; X64-NEXT: retq
%m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([513 x i8], ptr @.str, i32 0, i32 1), i64 2) nounwind
ret i32 %m
Expand All @@ -77,9 +74,8 @@ define i1 @length2_gt_const(ptr %X, ptr %Y) nounwind {
; X64-LABEL: length2_gt_const:
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: rolw $8, %ax
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: addl $-12594, %eax # imm = 0xCECE
; X64-NEXT: bswapl %eax
; X64-NEXT: addl $-825360384, %eax # imm = 0xCECE0000
; X64-NEXT: testl %eax, %eax
; X64-NEXT: setg %al
; X64-NEXT: retq
Expand All @@ -105,10 +101,8 @@ define i1 @length2_lt(ptr %X, ptr %Y) nounwind {
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: movzwl (%rsi), %ecx
; X64-NEXT: rolw $8, %ax
; X64-NEXT: rolw $8, %cx
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: movzwl %cx, %ecx
; X64-NEXT: bswapl %eax
; X64-NEXT: bswapl %ecx
; X64-NEXT: subl %ecx, %eax
; X64-NEXT: shrl $31, %eax
; X64-NEXT: # kill: def $al killed $al killed $eax
Expand All @@ -123,10 +117,8 @@ define i1 @length2_gt(ptr %X, ptr %Y) nounwind {
; X64: # %bb.0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: movzwl (%rsi), %ecx
; X64-NEXT: rolw $8, %ax
; X64-NEXT: rolw $8, %cx
; X64-NEXT: movzwl %ax, %eax
; X64-NEXT: movzwl %cx, %ecx
; X64-NEXT: bswapl %eax
; X64-NEXT: bswapl %ecx
; X64-NEXT: subl %ecx, %eax
; X64-NEXT: testl %eax, %eax
; X64-NEXT: setg %al
Expand Down Expand Up @@ -537,8 +529,8 @@ define i32 @length12(ptr %X, ptr %Y) nounwind {
; X64-NEXT: # %bb.1: # %loadbb1
; X64-NEXT: movl 8(%rdi), %ecx
; X64-NEXT: movl 8(%rsi), %edx
; X64-NEXT: bswapl %ecx
; X64-NEXT: bswapl %edx
; X64-NEXT: bswapq %rcx
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

bswapq is 2 uops vs 1 for bswapl on some CPU isn't it?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, on MOVBE targets this will no longer fold the loads.

; X64-NEXT: bswapq %rdx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: je .LBB31_3
Expand Down
Loading