Improve handling of disjunctions when comparing types

There were two problems in a comparison

    A | B <: C

 1. If the LHS was `a.type | Nothing`, atoms computation
    did not correctly account for Nothing.

 2. For other comparisons, the preliminary widen test was
    not done under a frozen constraint, and therefore cut
    off solutions. This caused a problem for the
    `summon[(1 | Nothing) <:< 1]` comparison even once (1)
    was fixed.

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -441,7 +441,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         def widenOK =
           (tp2.widenSingletons eq tp2)
           && (tp1.widenSingletons ne tp1)
-          && recurInFrozenGadt(tp1.widenSingletons, tp2)
+          && inFrozenConstraint(recurInFrozenGadt(tp1.widenSingletons, tp2))
 
         def joinOK = tp2.dealiasKeepRefiningAnnots match {
           case tp2: AppliedType if !tp2.tycon.typeSymbol.isClass =>

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -3104,7 +3104,10 @@ object Types {
 
     private def ensureAtomsComputed()(using Context): Unit =
       if atomsRunId != ctx.runId then
-        myAtoms = tp1.atoms | tp2.atoms
+        myAtoms =
+          if tp1.hasClassSymbol(defn.NothingClass) then tp2.atoms
+          else if tp2.hasClassSymbol(defn.NothingClass) then tp1.atoms
+          else tp1.atoms | tp2.atoms
         val tp1w = tp1.widenSingletons
         val tp2w = tp2.widenSingletons
         myWidened = if ((tp1 eq tp1w) && (tp2 eq tp2w)) this else tp1w | tp2w

tests/neg/singletonOrs.scala

@@ -24,7 +24,7 @@ object Get2As1 {
     def get2as11X:V = tuToV()(2)  // error
   }
   class Or11Nothing extends Or11X[Nothing]
-  def get2as1:1 = new Or11Nothing().get2as11X // error
+  def get2as1:1 = new Or11Nothing().get2as11X // ok
 
   def main(a:Array[String]) = {
     println(get2as1) // prints 2

tests/pos/i11003.scala

@@ -0,0 +1,20 @@
+object Foo
+
+val x: Foo.type = Foo
+val y: Foo.type | Nothing = x
+val z: Foo.type = y
+val a: 1 | Nothing = 1
+val b: 1 = a
+
+val intSuper = summon[(Int | Nothing) <:< Int]
+val intSub = summon[Int <:< (Int | Nothing)]
+val intEq = summon[Int =:= (Int | Nothing)]
+
+val fooSuper = summon[(Foo.type | Nothing) <:< Foo.type]
+val fooSub = summon[Foo.type <:< (Foo.type | Nothing)]
+val fooEq = summon[Foo.type =:= (Foo.type | Nothing)]
+
+val oneSuper = summon[(1 | Nothing) <:< 1]
+val oneSub = summon[1 <:< (1 | Nothing)]
+val oneEq = summon[1 =:= (1 | Nothing)]
+