Fix logic in isCheckDefinitelyFalse

Author: liufengyun

compiler/src/dotty/tools/dotc/transform/TypeTestsCasts.scala

@@ -157,8 +157,12 @@ object TypeTestsCasts {
               val xClass = effectiveClass(x)
               val pClass = effectiveClass(p)
 
-              !xClass.derivesFrom(pClass)
-              && (xClass.is(Final) || pClass.is(Final) || !xClass.is(Trait) && !pClass.is(Trait))
+              val bothAreClasses = !xClass.is(Trait) && !pClass.is(Trait)
+              val notXsubP = !xClass.derivesFrom(pClass)
+              val notPsubX = !pClass.derivesFrom(xClass)
+              bothAreClasses && notXsubP && notPsubX
+              || xClass.is(Final) && notXsubP
+              || pClass.is(Final) && notPsubX
             else
               false
     }
@@ -183,7 +187,8 @@ object TypeTestsCasts {
             val res1 = recur(tp1, P)
             val res2 = recur(tp2, P)
 
-            if res1.isEmpty && res2.isEmpty then res1
+            if res1.isEmpty && res2.isEmpty then
+              res1
             else if res2.isEmpty then
               if isCheckDefinitelyFalse(tp1, P) then res2
               else res1

tests/neg-custom-args/isInstanceOf/i5826.scala

@@ -19,7 +19,23 @@ class Foo {
   }
 
   def test4[A](x: List[Int] | (A => Int)) = x match {
-    case ls: List[Int] => ls.head       // ok, List decomposes to Some and None
+    case ls: List[Int] => ls.head       // error, List extends Int => T
+    case _ => 0
+  }
+
+  final class C[T] extends A[T]
+
+  def test5[T](x: A[T] | B[T] | Option[T]): Boolean = x.isInstanceOf[C[String]] // error
+
+  def test6[T](x: A[T] | B[T] | Option[T]): Boolean = x.isInstanceOf[C[T]]
+
+  def test7[A](x: Option[Int] | (A => Int)) = x match {
+    case ls: Option[Int] => ls.head       // OK, Option decomposes to Some and None
+    case _ => 0
+  }
+
+  def test8(x: List[Int] | A[String]) = x match {
+    case ls: List[Int] => ls.head        // OK, List decomposes to :: and Nil
     case _ => 0
   }
 }