Merge pull request #6333 from apple/address-non-null

Add a flag to `Address` and `Lvalue` that is used to keep track of whether the pointer is known not to be null

Author: ahatanaka

clang/lib/CodeGen/Address.h

@@ -22,64 +22,23 @@
 namespace clang {
 namespace CodeGen {
 
-// We try to save some space by using 6 bits over two PointerIntPairs to store
-// the alignment. However, some arches don't support 3 bits in a PointerIntPair
-// so we fallback to storing the alignment separately.
-template <typename T, bool = alignof(llvm::Value *) >= 8> class AddressImpl {};
-
-template <typename T> class AddressImpl<T, false> {
-  llvm::Value *Pointer;
-  llvm::Type *ElementType;
-  CharUnits Alignment;
-
-public:
-  AddressImpl(llvm::Value *Pointer, llvm::Type *ElementType,
-              CharUnits Alignment)
-      : Pointer(Pointer), ElementType(ElementType), Alignment(Alignment) {}
-  llvm::Value *getPointer() const { return Pointer; }
-  llvm::Type *getElementType() const { return ElementType; }
-  CharUnits getAlignment() const { return Alignment; }
-};
-
-template <typename T> class AddressImpl<T, true> {
-  // Int portion stores upper 3 bits of the log of the alignment.
-  llvm::PointerIntPair<llvm::Value *, 3, unsigned> Pointer;
-  // Int portion stores lower 3 bits of the log of the alignment.
-  llvm::PointerIntPair<llvm::Type *, 3, unsigned> ElementType;
-
-public:
-  AddressImpl(llvm::Value *Pointer, llvm::Type *ElementType,
-              CharUnits Alignment)
-      : Pointer(Pointer), ElementType(ElementType) {
-    if (Alignment.isZero())
-      return;
-    // Currently the max supported alignment is much less than 1 << 63 and is
-    // guaranteed to be a power of 2, so we can store the log of the alignment
-    // into 6 bits.
-    assert(Alignment.isPowerOfTwo() && "Alignment cannot be zero");
-    auto AlignLog = llvm::Log2_64(Alignment.getQuantity());
-    assert(AlignLog < (1 << 6) && "cannot fit alignment into 6 bits");
-    this->Pointer.setInt(AlignLog >> 3);
-    this->ElementType.setInt(AlignLog & 7);
-  }
-  llvm::Value *getPointer() const { return Pointer.getPointer(); }
-  llvm::Type *getElementType() const { return ElementType.getPointer(); }
-  CharUnits getAlignment() const {
-    unsigned AlignLog = (Pointer.getInt() << 3) | ElementType.getInt();
-    return CharUnits::fromQuantity(CharUnits::QuantityType(1) << AlignLog);
-  }
-};
+// Indicates whether a pointer is known not to be null.
+enum KnownNonNull_t { NotKnownNonNull, KnownNonNull };
 
 /// An aligned address.
 class Address {
-  AddressImpl<void> A;
+  llvm::PointerIntPair<llvm::Value *, 1, bool> PointerAndKnownNonNull;
+  llvm::Type *ElementType;
+  CharUnits Alignment;
 
 protected:
-  Address(std::nullptr_t) : A(nullptr, nullptr, CharUnits::Zero()) {}
+  Address(std::nullptr_t) : ElementType(nullptr) {}
 
 public:
-  Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment)
-      : A(Pointer, ElementType, Alignment) {
+  Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment,
+          KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
+      : PointerAndKnownNonNull(Pointer, IsKnownNonNull),
+        ElementType(ElementType), Alignment(Alignment) {
     assert(Pointer != nullptr && "Pointer cannot be null");
     assert(ElementType != nullptr && "Element type cannot be null");
     assert(llvm::cast<llvm::PointerType>(Pointer->getType())
@@ -88,11 +47,13 @@ class Address {
   }
 
   static Address invalid() { return Address(nullptr); }
-  bool isValid() const { return A.getPointer() != nullptr; }
+  bool isValid() const {
+    return PointerAndKnownNonNull.getPointer() != nullptr;
+  }
 
   llvm::Value *getPointer() const {
     assert(isValid());
-    return A.getPointer();
+    return PointerAndKnownNonNull.getPointer();
   }
 
   /// Return the type of the pointer value.
@@ -103,7 +64,7 @@ class Address {
   /// Return the type of the values stored in this address.
   llvm::Type *getElementType() const {
     assert(isValid());
-    return A.getElementType();
+    return ElementType;
   }
 
   /// Return the address space that this address resides in.
@@ -119,19 +80,35 @@ class Address {
   /// Return the alignment of this pointer.
   CharUnits getAlignment() const {
     assert(isValid());
-    return A.getAlignment();
+    return Alignment;
   }
 
   /// Return address with different pointer, but same element type and
   /// alignment.
-  Address withPointer(llvm::Value *NewPointer) const {
-    return Address(NewPointer, getElementType(), getAlignment());
+  Address withPointer(llvm::Value *NewPointer,
+                      KnownNonNull_t IsKnownNonNull) const {
+    return Address(NewPointer, getElementType(), getAlignment(),
+                   IsKnownNonNull);
   }
 
   /// Return address with different alignment, but same pointer and element
   /// type.
   Address withAlignment(CharUnits NewAlignment) const {
-    return Address(getPointer(), getElementType(), NewAlignment);
+    return Address(getPointer(), getElementType(), NewAlignment,
+                   isKnownNonNull());
+  }
+
+  /// Whether the pointer is known not to be null.
+  KnownNonNull_t isKnownNonNull() const {
+    assert(isValid());
+    return (KnownNonNull_t)PointerAndKnownNonNull.getInt();
+  }
+
+  /// Set the non-null bit.
+  Address setKnownNonNull() {
+    assert(isValid());
+    PointerAndKnownNonNull.setInt(true);
+    return *this;
   }
 };
 

clang/lib/CodeGen/CGBuilder.h

@@ -160,7 +160,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
     assert(cast<llvm::PointerType>(Ty)->isOpaqueOrPointeeTypeMatches(
                Addr.getElementType()) &&
            "Should not change the element type");
-    return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name));
+    return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name),
+                            Addr.isKnownNonNull());
   }
 
   /// Cast the element type of the given address to a different type,
@@ -169,7 +170,7 @@ class CGBuilderTy : public CGBuilderBaseTy {
                                const llvm::Twine &Name = "") {
     auto *PtrTy = Ty->getPointerTo(Addr.getAddressSpace());
     return Address(CreateBitCast(Addr.getPointer(), PtrTy, Name), Ty,
-                   Addr.getAlignment());
+                   Addr.getAlignment(), Addr.isKnownNonNull());
   }
 
   using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast;
@@ -178,7 +179,7 @@ class CGBuilderTy : public CGBuilderBaseTy {
                                               const llvm::Twine &Name = "") {
     llvm::Value *Ptr =
         CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name);
-    return Address(Ptr, ElementTy, Addr.getAlignment());
+    return Address(Ptr, ElementTy, Addr.getAlignment(), Addr.isKnownNonNull());
   }
 
   /// Given
@@ -199,7 +200,7 @@ class CGBuilderTy : public CGBuilderBaseTy {
     return Address(
         CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name),
         ElTy->getElementType(Index),
-        Addr.getAlignment().alignmentAtOffset(Offset));
+        Addr.getAlignment().alignmentAtOffset(Offset), Addr.isKnownNonNull());
   }
 
   /// Given
@@ -221,7 +222,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
         CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
                           {getSize(CharUnits::Zero()), getSize(Index)}, Name),
         ElTy->getElementType(),
-        Addr.getAlignment().alignmentAtOffset(Index * EltSize));
+        Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+        Addr.isKnownNonNull());
   }
 
   /// Given
@@ -237,8 +239,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
 
     return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
                                      getSize(Index), Name),
-                   ElTy,
-                   Addr.getAlignment().alignmentAtOffset(Index * EltSize));
+                   ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+                   Addr.isKnownNonNull());
   }
 
   /// Given
@@ -255,7 +257,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
     return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(),
                              getSize(Index), Name),
                    Addr.getElementType(),
-                   Addr.getAlignment().alignmentAtOffset(Index * EltSize));
+                   Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+                   NotKnownNonNull);
   }
 
   /// Create GEP with single dynamic index. The address alignment is reduced
@@ -270,7 +273,7 @@ class CGBuilderTy : public CGBuilderBaseTy {
     return Address(
         CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, Name),
         Addr.getElementType(),
-        Addr.getAlignment().alignmentOfArrayElement(EltSize));
+        Addr.getAlignment().alignmentOfArrayElement(EltSize), NotKnownNonNull);
   }
 
   /// Given a pointer to i8, adjust it by a given constant offset.
@@ -280,15 +283,17 @@ class CGBuilderTy : public CGBuilderBaseTy {
     return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
                                      getSize(Offset), Name),
                    Addr.getElementType(),
-                   Addr.getAlignment().alignmentAtOffset(Offset));
+                   Addr.getAlignment().alignmentAtOffset(Offset),
+                   Addr.isKnownNonNull());
   }
   Address CreateConstByteGEP(Address Addr, CharUnits Offset,
                              const llvm::Twine &Name = "") {
     assert(Addr.getElementType() == TypeCache.Int8Ty);
     return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(),
                              getSize(Offset), Name),
                    Addr.getElementType(),
-                   Addr.getAlignment().alignmentAtOffset(Offset));
+                   Addr.getAlignment().alignmentAtOffset(Offset),
+                   NotKnownNonNull);
   }
 
   using CGBuilderBaseTy::CreateConstInBoundsGEP2_32;
@@ -305,7 +310,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
       llvm_unreachable("offset of GEP with constants is always computable");
     return Address(GEP, GEP->getResultElementType(),
                    Addr.getAlignment().alignmentAtOffset(
-                       CharUnits::fromQuantity(Offset.getSExtValue())));
+                       CharUnits::fromQuantity(Offset.getSExtValue())),
+                   Addr.isKnownNonNull());
   }
 
   using CGBuilderBaseTy::CreateMemCpy;
@@ -369,7 +375,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
 
   using CGBuilderBaseTy::CreateLaunderInvariantGroup;
   Address CreateLaunderInvariantGroup(Address Addr) {
-    return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()));
+    return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()),
+                            Addr.isKnownNonNull());
   }
 };
 

clang/lib/CodeGen/CGCall.cpp

@@ -2791,7 +2791,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
     case ABIArgInfo::IndirectAliased: {
       assert(NumIRArgs == 1);
       Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty),
-                                  ArgI.getIndirectAlign());
+                                  ArgI.getIndirectAlign(), KnownNonNull);
 
       if (!hasScalarEvaluationKind(Ty)) {
         // Aggregates and complex variables are accessed by reference. All we

clang/lib/CodeGen/CGClass.cpp

@@ -138,7 +138,7 @@ Address CodeGenFunction::LoadCXXThisAddress() {
   }
 
   llvm::Type *Ty = ConvertType(MD->getThisType()->getPointeeType());
-  return Address(LoadCXXThis(), Ty, CXXThisAlignment);
+  return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull);
 }
 
 /// Emit the address of a field using a member data pointer.
@@ -391,7 +391,7 @@ Address CodeGenFunction::GetAddressOfBaseClass(
     llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
     PHI->addIncoming(Value.getPointer(), notNullBB);
     PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
-    Value = Value.withPointer(PHI);
+    Value = Value.withPointer(PHI, NotKnownNonNull);
   }
 
   return Value;

clang/lib/CodeGen/CGDecl.cpp

@@ -2503,8 +2503,10 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
              CGM.getDataLayout().getAllocaAddrSpace());
       auto DestAS = getContext().getTargetAddressSpace(DestLangAS);
       auto *T = DeclPtr.getElementType()->getPointerTo(DestAS);
-      DeclPtr = DeclPtr.withPointer(getTargetHooks().performAddrSpaceCast(
-          *this, V, SrcLangAS, DestLangAS, T, true));
+      DeclPtr =
+          DeclPtr.withPointer(getTargetHooks().performAddrSpaceCast(
+                                  *this, V, SrcLangAS, DestLangAS, T, true),
+                              DeclPtr.isKnownNonNull());
     }
 
     // For truly ABI indirect arguments -- those that are not `byval` -- store

clang/lib/CodeGen/CGException.cpp

@@ -1845,7 +1845,7 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
   llvm::Value *ChildVar =
       Builder.CreateBitCast(RecoverCall, ParentVar.getType());
   ChildVar->setName(ParentVar.getName());
-  return ParentVar.withPointer(ChildVar);
+  return ParentVar.withPointer(ChildVar, KnownNonNull);
 }
 
 void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,