[Serialization] (De-)serialize witnesses completely.

The witnesses in a NormalProtocolConformance have never been
completely serialized, because their substitutions involved a weird
mix of archetypes that blew up the deserialization code. So, only the
witness declarations themselves got serialized. Many clients (the type
checker, SourceKit, etc.) didn't need the extra information, but some
clients (e.g., the SIL optimizers) would end up recomputing this
information. Ick.

Now, serialize the complete Witness structure along with the AST,
including information about the synthetic environment, complete
substitutions, etc. This should obsolete some redundant code paths in
the SIL optimization infrastructure.

This (de-)serialization code takes a new-ish approach to serializing
the synthetic environment in that it avoids serializing any
archetypes. Rather, it maps everything back to interface types during
serialization, and deserialization forms a new generic environment
(with new archetypes!) on-the-fly, mapping deserialized types back
into that environment (and to those archetypes). This way, we don't
have to maintain identity of archetypes in the deserialization code,
and might get some better re-use of the archetypes.

More of rdar://problem/24079818.

Author: DougGregor

include/swift/AST/SubstitutionMap.h

@@ -56,6 +56,9 @@ class SubstitutionMap {
     return subMap;
   }
 
+  /// Retrieve the conformances for the given type.
+  ArrayRef<ProtocolConformanceRef> getConformances(CanType type) const;
+
   void addSubstitution(CanType type, Type replacement);
 
   void addConformances(CanType type, ArrayRef<ProtocolConformanceRef> conformances);

include/swift/AST/Witness.h

@@ -81,7 +81,7 @@ class ValueDecl;
 /// environment are those of the context of the protocol conformance (\c U
 /// and \c V, in the example above) and the innermost generic parameters are
 /// those of the generic requirement (\c T, in the example above). The
-/// \c Witness class contains this syntheetic environment (both its generic
+/// \c Witness class contains this synthetic environment (both its generic
 /// signature and a generic environment providing archetypes), a substitution
 /// map that allows one to map the interface types of the requirement into
 /// the interface types of the synthetic domain, and the set of substitutions

include/swift/Serialization/ModuleFile.h

@@ -710,10 +710,14 @@ class ModuleFile : public LazyMemberLoader {
   /// Reads a substitution record from \c DeclTypeCursor.
   ///
   /// If the record at the cursor is not a substitution, returns None.
-  Optional<Substitution> maybeReadSubstitution(llvm::BitstreamCursor &Cursor);
+  Optional<Substitution> maybeReadSubstitution(llvm::BitstreamCursor &Cursor,
+                                               GenericEnvironment *genericEnv =
+                                                nullptr);
 
   /// Recursively reads a protocol conformance from the given cursor.
-  ProtocolConformanceRef readConformance(llvm::BitstreamCursor &Cursor);
+  ProtocolConformanceRef readConformance(llvm::BitstreamCursor &Cursor,
+                                         GenericEnvironment *genericEnv =
+                                           nullptr);
 
   /// Read the given normal conformance from the current module file.
   NormalProtocolConformance *

include/swift/Serialization/ModuleFormat.h

@@ -54,7 +54,7 @@ const uint16_t VERSION_MAJOR = 0;
 /// in source control, you should also update the comment to briefly
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
-const uint16_t VERSION_MINOR = 280; // Last change: function qualified ownership
+const uint16_t VERSION_MINOR = 281; // Last change: complete witnesses
 
 using DeclID = PointerEmbeddedInt<unsigned, 31>;
 using DeclIDField = BCFixed<31>;

lib/AST/ASTVerifier.cpp

@@ -1893,7 +1893,7 @@ struct ASTNodeBase {};
           }
 
           // Check the witness substitutions.
-          auto witness = normal->getWitness(req, nullptr);
+          const auto &witness = normal->getWitness(req, nullptr);
 
           if (witness.requiresSubstitution()) {
             GenericEnv.push_back(witness.getSyntheticEnvironment());

lib/AST/SubstitutionMap.cpp

@@ -106,6 +106,13 @@ addConformances(CanType type, ArrayRef<ProtocolConformanceRef> conformances) {
   (void) result;
 }
 
+ArrayRef<ProtocolConformanceRef> SubstitutionMap::
+getConformances(CanType type) const {
+  auto known = conformanceMap.find(type.getPointer());
+  if (known == conformanceMap.end()) return { };
+  return known->second;
+}
+
 void SubstitutionMap::
 addParent(CanType type, CanType parent, AssociatedTypeDecl *assocType) {
   assert(type && parent && assocType);

lib/Serialization/Deserialization.cpp

@@ -450,7 +450,9 @@ Pattern *ModuleFile::maybeReadPattern() {
   }
 }
 
-ProtocolConformanceRef ModuleFile::readConformance(llvm::BitstreamCursor &Cursor){
+ProtocolConformanceRef ModuleFile::readConformance(
+                                             llvm::BitstreamCursor &Cursor,
+                                             GenericEnvironment *genericEnv) {
   using namespace decls_block;
 
   SmallVector<uint64_t, 16> scratch;
@@ -475,6 +477,10 @@ ProtocolConformanceRef ModuleFile::readConformance(llvm::BitstreamCursor &Cursor
 
     ASTContext &ctx = getContext();
     Type conformingType = getType(conformingTypeID);
+    if (genericEnv) {
+      conformingType = genericEnv->mapTypeIntoContext(getAssociatedModule(),
+                                                      conformingType);
+    }
 
     PrettyStackTraceType trace(getAssociatedModule()->getASTContext(),
                                "reading specialized conformance for",
@@ -483,12 +489,13 @@ ProtocolConformanceRef ModuleFile::readConformance(llvm::BitstreamCursor &Cursor
     // Read the substitutions.
     SmallVector<Substitution, 4> substitutions;
     while (numSubstitutions--) {
-      auto sub = maybeReadSubstitution(Cursor);
+      auto sub = maybeReadSubstitution(Cursor, genericEnv);
       assert(sub.hasValue() && "Missing substitution?");
       substitutions.push_back(*sub);
     }
 
-    ProtocolConformanceRef genericConformance = readConformance(Cursor);
+    ProtocolConformanceRef genericConformance =
+      readConformance(Cursor, genericEnv);
     PrettyStackTraceDecl traceTo("... to", genericConformance.getRequirement());
 
     assert(genericConformance.isConcrete() && "Abstract generic conformance?");
@@ -505,11 +512,17 @@ ProtocolConformanceRef ModuleFile::readConformance(llvm::BitstreamCursor &Cursor
 
     ASTContext &ctx = getContext();
     Type conformingType = getType(conformingTypeID);
+    if (genericEnv) {
+      conformingType = genericEnv->mapTypeIntoContext(getAssociatedModule(),
+                                                      conformingType);
+    }
+
     PrettyStackTraceType trace(getAssociatedModule()->getASTContext(),
                                "reading inherited conformance for",
                                conformingType);
 
-    ProtocolConformanceRef inheritedConformance = readConformance(Cursor);
+    ProtocolConformanceRef inheritedConformance =
+      readConformance(Cursor, genericEnv);
     PrettyStackTraceDecl traceTo("... to",
                                  inheritedConformance.getRequirement());
 
@@ -541,8 +554,7 @@ ProtocolConformanceRef ModuleFile::readConformance(llvm::BitstreamCursor &Cursor
     auto module = getModule(moduleID);
 
     SmallVector<ProtocolConformance *, 2> conformances;
-    nominal->lookupConformance(module, proto,
-                               conformances);
+    nominal->lookupConformance(module, proto, conformances);
     PrettyStackTraceModuleFile traceMsg(
         "If you're seeing a crash here, check that your SDK and dependencies "
         "are at least as new as the versions used to build", this);
@@ -641,7 +653,8 @@ NormalProtocolConformance *ModuleFile::readNormalConformance(
 }
 
 Optional<Substitution>
-ModuleFile::maybeReadSubstitution(llvm::BitstreamCursor &cursor) {
+ModuleFile::maybeReadSubstitution(llvm::BitstreamCursor &cursor,
+                                  GenericEnvironment *genericEnv) {
   BCOffsetRAII lastRecordOffset(cursor);
 
   auto entry = cursor.advance(AF_DontPopBlockAtEnd);
@@ -661,6 +674,10 @@ ModuleFile::maybeReadSubstitution(llvm::BitstreamCursor &cursor) {
                                                           numConformances);
 
   auto replacementTy = getType(replacementID);
+  if (genericEnv) {
+    replacementTy = genericEnv->mapTypeIntoContext(getAssociatedModule(),
+                                                   replacementTy);
+  }
 
   SmallVector<ProtocolConformanceRef, 4> conformanceBuf;
   while (numConformances--) {
@@ -4206,17 +4223,85 @@ void ModuleFile::finishNormalConformance(NormalProtocolConformance *conformance,
   ASTContext &ctx = getContext();
   ArrayRef<uint64_t>::iterator rawIDIter = rawIDs.begin();
 
-  WitnessMap witnesses;
   while (valueCount--) {
-    auto first = cast<ValueDecl>(getDecl(*rawIDIter++));
-    auto second = cast_or_null<ValueDecl>(getDecl(*rawIDIter++));
-    assert(second || first->getAttrs().hasAttribute<OptionalAttr>() ||
-           first->getAttrs().isUnavailable(ctx));
-    (void) ctx;
-    if (second)
-      witnesses.insert(std::make_pair(first, second));
-    else
-      witnesses.insert(std::make_pair(first, Witness()));
+    auto req = cast<ValueDecl>(getDecl(*rawIDIter++));
+    auto witness = cast_or_null<ValueDecl>(getDecl(*rawIDIter++));
+    assert(witness ||
+           req->getAttrs().hasAttribute<OptionalAttr>() ||
+           req->getAttrs().isUnavailable(ctx));
+    if (!witness) {
+      conformance->setWitness(req, Witness());
+      continue;
+    }
+
+    // Generic signature and environment.
+    GenericSignature *syntheticSig = nullptr;
+    GenericEnvironment *syntheticEnv = nullptr;
+    if (unsigned numGenericParams = *rawIDIter++) {
+      // Generic parameters.
+      SmallVector<GenericTypeParamType *, 2> genericParams;
+      while (numGenericParams--) {
+        genericParams.push_back(
+          getType(*rawIDIter++)->castTo<GenericTypeParamType>());
+      }
+
+      // Generic requirements.
+      SmallVector<Requirement, 4> requirements;
+      readGenericRequirements(requirements);
+
+      // Form the generic signature for the synthetic environment.
+      syntheticSig = GenericSignature::get(genericParams, requirements);
+
+      // Create an archetype builder, which will help us create the
+      // synthetic environment.
+      ArchetypeBuilder builder(*getAssociatedModule(), ctx.Diags);
+      builder.addGenericSignature(syntheticSig, nullptr);
+      builder.finalize(SourceLoc());
+      syntheticEnv = builder.getGenericEnvironment();
+    }
+
+    // Requirement -> synthetic map.
+    SubstitutionMap reqToSyntheticMap;
+    bool hasReqToSyntheticMap = false;
+    if (unsigned numEntries = *rawIDIter++) {
+      hasReqToSyntheticMap = true;
+      while (numEntries--) {
+        auto first = getType(*rawIDIter++);
+        auto second = getType(*rawIDIter++);
+        reqToSyntheticMap.addSubstitution(first->getCanonicalType(), second);
+
+        if (unsigned numConformances = *rawIDIter++) {
+          SmallVector<ProtocolConformanceRef, 2> conformances;
+          while (numConformances--) {
+            conformances.push_back(readConformance(DeclTypeCursor));
+          }
+          reqToSyntheticMap.addConformances(first->getCanonicalType(),
+                                            ctx.AllocateCopy(conformances));
+        }
+      }
+    }
+
+    // Witness substitutions.
+    SmallVector<Substitution, 4> witnessSubstitutions;
+    if (unsigned numWitnessSubstitutions = *rawIDIter++) {
+      while (numWitnessSubstitutions--) {
+        auto sub = maybeReadSubstitution(DeclTypeCursor, syntheticEnv);
+        witnessSubstitutions.push_back(*sub);
+      }
+    }
+
+    // Handle simple witnesses.
+    if (witnessSubstitutions.empty() && !syntheticSig && !syntheticEnv &&
+        !hasReqToSyntheticMap) {
+      conformance->setWitness(req, Witness(witness));
+      continue;
+    }
+
+    // Set the witness.
+    conformance->setWitness(req,
+                            Witness(witness, witnessSubstitutions,
+                                    syntheticSig, syntheticEnv,
+                                    reqToSyntheticMap));
   }
   assert(rawIDIter <= rawIDs.end() && "read too much");
 
@@ -4232,16 +4317,12 @@ void ModuleFile::finishNormalConformance(NormalProtocolConformance *conformance,
   }
   assert(rawIDIter <= rawIDs.end() && "read too much");
 
+
   // Set type witnesses.
   for (auto typeWitness : typeWitnesses) {
     conformance->setTypeWitness(typeWitness.first, typeWitness.second.first,
                                 typeWitness.second.second);
   }
-
-  // Set witnesses.
-  for (auto witness : witnesses) {
-    conformance->setWitness(witness.first, witness.second);
-  }
 }
 
 static Optional<ForeignErrorConvention::Kind>