Give multi-payload enums extra inhabitants.

Previously, they would forward their unused spare bits to be used by other multi-payload enums, but
did not implement anything for single-payload extra inhabitants.

Author: jckarter

include/swift/ABI/Enum.h

@@ -0,0 +1,53 @@
+//===--- Enum.h - Enum implementation runtime declarations ------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// Enum implementation details declarations relevant to the ABI.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_ABI_ENUM_H
+#define SWIFT_ABI_ENUM_H
+
+#include <stdlib.h>
+
+namespace swift {
+
+struct EnumTagCounts {
+  unsigned numTags, numTagBytes;
+};
+
+inline EnumTagCounts
+getEnumTagCounts(size_t size, unsigned emptyCases, unsigned payloadCases) {
+  // We can use the payload area with a tag bit set somewhere outside of the
+  // payload area to represent cases. See how many bytes we need to cover
+  // all the empty cases.
+  unsigned numTags = payloadCases;
+  if (emptyCases > 0) {
+    if (size >= 4)
+      // Assume that one tag bit is enough if the precise calculation overflows
+      // an int32.
+      numTags += 1;
+    else {
+      unsigned bits = size * 8U;
+      unsigned casesPerTagBitValue = 1U << bits;
+      numTags += ((emptyCases + (casesPerTagBitValue-1U)) >> bits);
+    }
+  }
+  unsigned numTagBytes = (numTags <=    1 ? 0 :
+                          numTags <   256 ? 1 :
+                          numTags < 65536 ? 2 : 4);
+  return {numTags, numTagBytes};
+}
+
+} // namespace swift
+
+#endif // SWIFT_ABI_ENUM_H

lib/IRGen/GenEnum.cpp

@@ -1181,7 +1181,7 @@ namespace {
     // discriminate enum cases.
     unsigned ExtraTagBitCount = ~0u;
     // The number of possible values for the extra tag bits that are used.
-    // Log2(NumExtraTagValues - 1) + 1 == ExtraTagBitCount
+    // Log2(NumExtraTagValues - 1) + 1 <= ExtraTagBitCount
     unsigned NumExtraTagValues = ~0u;
 
     APInt getExtraTagBitConstant(uint64_t value) const {
@@ -4820,41 +4820,161 @@ namespace {
 
     /// \group Extra inhabitants
 
-    // TODO
+    // If we didn't use all of the available tag bit representations, offer
+    // the remaining ones as extra inhabitants.
 
-    bool mayHaveExtraInhabitants(IRGenModule &) const override { return false; }
+    bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
+      if (TIK >= Fixed)
+        return getFixedExtraInhabitantCount(IGM) > 0;
+      return true;
+    }
+    
+    /// Rounds the extra tag bit count up to the next byte size.
+    unsigned getExtraTagBitCountForExtraInhabitants() const {
+      if (!ExtraTagTy)
+        return 0;
+      return (ExtraTagTy->getBitWidth() + 7) & ~7;
+    }
+    
+    Address projectExtraTagBitsForExtraInhabitants(IRGenFunction &IGF,
+                                                   Address base) const {
+      auto addr = projectExtraTagBits(IGF, base);
+      if (ExtraTagTy->getBitWidth() != getExtraTagBitCountForExtraInhabitants()) {
+        addr = IGF.Builder.CreateBitCast(addr,
+             llvm::IntegerType::get(IGF.IGM.getLLVMContext(),
+                                     getExtraTagBitCountForExtraInhabitants())
+               ->getPointerTo());
+      }
+      return addr;
+    }
 
     llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
                                          Address src,
                                          SILType T,
                                          bool isOutlined) const override {
-      llvm_unreachable("extra inhabitants for multi-payload enums not implemented");
+      // For dynamic layouts, the runtime provides a value witness to do this.
+      if (TIK < Fixed) {
+        return emitGetExtraInhabitantIndexCall(IGF, T, src);
+      }
+      
+      llvm::Value *tag;
+      if (CommonSpareBits.count()) {
+        auto payload = EnumPayload::load(IGF, projectPayload(IGF, src),
+                                         PayloadSchema);
+        tag = payload.emitGatherSpareBits(IGF, CommonSpareBits, 0, 32);
+        if (getExtraTagBitCountForExtraInhabitants()) {
+          auto extraTagAddr = projectExtraTagBitsForExtraInhabitants(IGF, src);
+          auto extraTag = IGF.Builder.CreateLoad(extraTagAddr);
+          auto extraTagBits =
+            IGF.Builder.CreateZExtOrTrunc(extraTag, IGF.IGM.Int32Ty);
+          extraTagBits =
+            IGF.Builder.CreateShl(extraTagBits, CommonSpareBits.count());
+          tag = IGF.Builder.CreateOr(tag, extraTagBits);
+        }
+      } else {
+        auto extraTagAddr = projectExtraTagBitsForExtraInhabitants(IGF, src);
+        auto extraTag = IGF.Builder.CreateLoad(extraTagAddr);
+        tag = IGF.Builder.CreateZExtOrTrunc(extraTag, IGF.IGM.Int32Ty);
+      }
+      
+      // Check whether it really is an extra inhabitant.
+      auto tagBits = CommonSpareBits.count() + getExtraTagBitCountForExtraInhabitants();
+      auto maxTag = tagBits >= 32 ? ~0u : (1 << tagBits) - 1;
+      auto index = IGF.Builder.CreateSub(
+                               llvm::ConstantInt::get(IGF.IGM.Int32Ty, maxTag),
+                               tag);
+      auto isExtraInhabitant = IGF.Builder.CreateICmpULT(index,
+                 llvm::ConstantInt::get(IGF.IGM.Int32Ty,
+                                        getFixedExtraInhabitantCount(IGF.IGM)));
+      return IGF.Builder.CreateSelect(isExtraInhabitant,
+                            index, llvm::ConstantInt::get(IGF.IGM.Int32Ty, -1));
     }
 
     void storeExtraInhabitant(IRGenFunction &IGF,
                               llvm::Value *index,
                               Address dest,
                               SILType T,
                               bool isOutlined) const override {
-      llvm_unreachable("extra inhabitants for multi-payload enums not implemented");
+      // For dynamic layouts, the runtime provides a value witness to do this.
+      if (TIK < Fixed) {
+        emitStoreExtraInhabitantCall(IGF, T, index, dest);
+        return;
+      }
+
+      auto indexValue = IGF.Builder.CreateNot(index);
+      if (CommonSpareBits.count()) {
+        // Factor the index value into parts to scatter into the payload and
+        // to store in the extra tag bits, if any.
+        EnumPayload payload =
+          interleaveSpareBits(IGF, PayloadSchema, CommonSpareBits, indexValue);
+        payload.store(IGF, projectPayload(IGF, dest));
+        if (getExtraTagBitCountForExtraInhabitants() > 0) {
+          auto tagBits = IGF.Builder.CreateLShr(indexValue,
+              llvm::ConstantInt::get(IGF.IGM.Int32Ty, CommonSpareBits.count()));
+          auto tagAddr = projectExtraTagBitsForExtraInhabitants(IGF, dest);
+          tagBits = IGF.Builder.CreateZExtOrTrunc(tagBits,
+                      tagAddr.getAddress()->getType()->getPointerElementType());
+          IGF.Builder.CreateStore(tagBits, tagAddr);
+        }
+      } else {
+        // Only need to store the tag value.
+        auto tagAddr = projectExtraTagBitsForExtraInhabitants(IGF, dest);
+        indexValue = IGF.Builder.CreateZExtOrTrunc(indexValue,
+                      tagAddr.getAddress()->getType()->getPointerElementType());
+        IGF.Builder.CreateStore(indexValue, tagAddr);
+      }
     }
 
     APInt
     getFixedExtraInhabitantMask(IRGenModule &IGM) const override {
-      // TODO may not always be all-ones
-      return APInt::getAllOnesValue(
-                      cast<FixedTypeInfo>(TI)->getFixedSize().getValueInBits());
+      // The extra inhabitant goes into the tag bits.
+      auto tagBits = CommonSpareBits.asAPInt();
+      auto fixedTI = cast<FixedTypeInfo>(TI);
+      if (getExtraTagBitCountForExtraInhabitants() > 0) {
+        auto bitSize = fixedTI->getFixedSize().getValueInBits();
+        tagBits = tagBits.zext(bitSize);
+        auto extraTagMask = APInt::getAllOnesValue(bitSize)
+          .shl(CommonSpareBits.size());
+        tagBits |= extraTagMask;
+      }
+      return tagBits;
     }
 
     unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
-      return 0;
+      unsigned totalTagBits = CommonSpareBits.count() + getExtraTagBitCountForExtraInhabitants();
+      if (totalTagBits >= 32)
+        return INT_MAX;
+      unsigned totalTags = 1u << totalTagBits;
+      return totalTags - ElementsWithPayload.size() - NumEmptyElementTags;
     }
 
     APInt
     getFixedExtraInhabitantValue(IRGenModule &IGM,
                                  unsigned bits,
                                  unsigned index) const override {
-      llvm_unreachable("extra inhabitants for multi-payload enums not implemented");
+      // Count down from all-ones since a small negative number constant is
+      // likely to be easier to reify.
+      auto mask = ~index;
+      auto extraTagMask = getExtraTagBitCountForExtraInhabitants() >= 32
+        ? ~0u : (1 << getExtraTagBitCountForExtraInhabitants()) - 1;
+
+      if (auto payloadBitCount = CommonSpareBits.count()) {
+        auto payloadTagMask = payloadBitCount >= 32
+          ? ~0u : (1 << payloadBitCount) - 1;
+        auto payloadPart = mask & payloadTagMask;
+        auto payloadBits = interleaveSpareBits(IGM, CommonSpareBits,
+                                               bits, payloadPart, 0);
+        if (getExtraTagBitCountForExtraInhabitants() > 0) {
+          auto extraBits = APInt(bits,
+                                 (mask >> payloadBitCount) & extraTagMask)
+            .shl(CommonSpareBits.size());
+          payloadBits |= extraBits;
+        }
+        return payloadBits;
+      } else {
+        auto value = APInt(bits, mask & extraTagMask);
+        return value.shl(CommonSpareBits.size());
+      }
     }
 
     ClusteredBitVector

stdlib/public/Reflection/TypeLowering.cpp

@@ -18,6 +18,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "swift/ABI/Enum.h"
 #include "swift/Reflection/TypeLowering.h"
 #include "swift/Reflection/TypeRef.h"
 #include "swift/Reflection/TypeRefBuilder.h"
@@ -948,31 +949,6 @@ class HasSingletonMetatype
   }
 };
 
-// Copy-and-pasted from stdlib/public/runtime/Enum.cpp -- should probably go
-// in a header somewhere, since the formula is part of the ABI.
-static unsigned getNumTagBytes(size_t size, unsigned emptyCases,
-                               unsigned payloadCases) {
-  // We can use the payload area with a tag bit set somewhere outside of the
-  // payload area to represent cases. See how many bytes we need to cover
-  // all the empty cases.
-
-  unsigned numTags = payloadCases;
-  if (emptyCases > 0) {
-    if (size >= 4)
-      // Assume that one tag bit is enough if the precise calculation overflows
-      // an int32.
-      numTags += 1;
-    else {
-      unsigned bits = size * 8U;
-      unsigned casesPerTagBitValue = 1U << bits;
-      numTags += ((emptyCases + (casesPerTagBitValue-1U)) >> bits);
-    }
-  }
-  return (numTags <=    1 ? 0 :
-          numTags <   256 ? 1 :
-          numTags < 65536 ? 2 : 4);
-}
-
 class EnumTypeInfoBuilder {
   TypeConverter &TC;
   unsigned Size, Alignment, NumExtraInhabitants;
@@ -1035,9 +1011,9 @@ class EnumTypeInfoBuilder {
     // NoPayloadEnumImplStrategy
     if (PayloadCases.empty()) {
       Kind = RecordKind::NoPayloadEnum;
-      Size += getNumTagBytes(/*size=*/0,
-                             NoPayloadCases,
-                             /*payloadCases=*/0);
+      Size += getEnumTagCounts(/*size=*/0,
+                               NoPayloadCases,
+                               /*payloadCases=*/0).numTagBytes;
 
     // SinglePayloadEnumImplStrategy
     } else if (PayloadCases.size() == 1) {
@@ -1065,9 +1041,9 @@ class EnumTypeInfoBuilder {
           // Not enough extra inhabitants for all cases. We have to add an
           // extra tag field.
           NumExtraInhabitants = 0;
-          Size += getNumTagBytes(Size,
-                                 NoPayloadCases - NumExtraInhabitants,
-                                 /*payloadCases=*/1);
+          Size += getEnumTagCounts(Size,
+                                   NoPayloadCases - NumExtraInhabitants,
+                                   /*payloadCases=*/1).numTagBytes;
         }
       }
 
@@ -1091,14 +1067,20 @@ class EnumTypeInfoBuilder {
         NumExtraInhabitants = FixedDescriptor->NumExtraInhabitants;
         BitwiseTakable = FixedDescriptor->isBitwiseTakable();
       } else {
-        // Dynamic multi-payload enums do not have extra inhabitants
-        NumExtraInhabitants = 0;
-
         // Dynamic multi-payload enums always use an extra tag to differentiate
         // between cases
-        Size += getNumTagBytes(Size,
-                               NoPayloadCases,
-                               PayloadCases.size());
+        auto tagCounts = getEnumTagCounts(Size, NoPayloadCases,
+                                          PayloadCases.size());
+        
+        Size += tagCounts.numTagBytes;
+        // Dynamic multi-payload enums use the tag representations not assigned
+        // to cases for extra inhabitants.
+        if (tagCounts.numTagBytes >= 32) {
+          NumExtraInhabitants = INT_MAX;
+        } else {
+          NumExtraInhabitants =
+            (1 << (tagCounts.numTagBytes * 8)) - tagCounts.numTags;
+        }
       }
     }