[InstCombine] Fix infinite min/max canonicalization loop (PR44541)

While D72944 also fixes https://bugs.llvm.org/show_bug.cgi?id=44541,
it does so in a more roundabout manner and there might be other
loopholes to trigger the same issue. This is a more direct fix,
that prevents the transform if the min/max is based on a
non-canonical sub X, 0 instruction.

Differential Revision: https://reviews.llvm.org/D73849

Author: nikic

llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp

@@ -1013,6 +1013,12 @@ canonicalizeMinMaxWithConstant(SelectInst &Sel, ICmpInst &Cmp,
       Cmp.getPredicate() == CanonicalPred)
     return nullptr;
 
+  // Bail out on unsimplified X-0 operand (due to some worklist management bug),
+  // as this may cause an infinite combine loop. Let the sub be folded first.
+  if (match(LHS, m_Sub(m_Value(), m_Zero())) ||
+      match(RHS, m_Sub(m_Value(), m_Zero())))
+    return nullptr;
+
   // Create the canonical compare and plug it into the select.
   Sel.setCondition(Builder.CreateICmp(CanonicalPred, LHS, RHS));
 

llvm/test/Transforms/InstCombine/pr44541.ll

@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine -expensive-combines=0 -instcombine-infinite-loop-threshold=3 < %s | FileCheck %s
+
+; This test used to cause an infinite combine loop.
+
+define i16 @passthru(i16 returned %x) {
+; CHECK-LABEL: @passthru(
+; CHECK-NEXT:    ret i16 [[X:%.*]]
+;
+  ret i16 %x
+}
+
+define i16 @test(i16 %arg) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[ZERO:%.*]] = call i16 @passthru(i16 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 [[ARG:%.*]], 0
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[TMP1]], i16 [[ARG]], i16 0
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %zero = call i16 @passthru(i16 0)
+  %sub = sub nuw nsw i16 %arg, %zero
+  %cmp = icmp slt i16 %sub, 0
+  %ret = select i1 %cmp, i16 0, i16 %sub
+  ret i16 %ret
+}