Merge branch 'autodiff-accessor-derivatives' of https://github.com/apple/swift into autodiff-accessor-derivatives

Author: ematejska

CMakeLists.txt

@@ -487,16 +487,6 @@ if(NOT SWIFT_LIPO)
   find_toolchain_tool(SWIFT_LIPO "${SWIFT_DARWIN_XCRUN_TOOLCHAIN}" lipo)
 endif()
 
-# Reset CMAKE_SYSTEM_PROCESSOR if not cross-compiling.
-# CMake refuses to use `uname -m` on OS X
-# http://public.kitware.com/Bug/view.php?id=10326
-if(NOT CMAKE_CROSSCOMPILING AND CMAKE_SYSTEM_PROCESSOR STREQUAL "i386")
-  execute_process(
-      COMMAND "uname" "-m"
-      OUTPUT_VARIABLE CMAKE_SYSTEM_PROCESSOR
-      OUTPUT_STRIP_TRAILING_WHITESPACE)
-endif()
-
 get_filename_component(SWIFT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} REALPATH)
 set(SWIFT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(SWIFT_CMAKE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")

benchmark/single-source/Substring.swift

@@ -17,6 +17,8 @@ public let SubstringTest = [
   BenchmarkInfo(name: "EqualSubstringString", runFunction: run_EqualSubstringString, tags: [.validation, .api, .String]),
   BenchmarkInfo(name: "EqualSubstringSubstring", runFunction: run_EqualSubstringSubstring, tags: [.validation, .api, .String]),
   BenchmarkInfo(name: "EqualSubstringSubstringGenericEquatable", runFunction: run_EqualSubstringSubstringGenericEquatable, tags: [.validation, .api, .String]),
+  BenchmarkInfo(name: "SubstringRemoveFirst1", runFunction: run_SubstringRemoveFirst1, tags: [.validation, .api, .String]),
+  BenchmarkInfo(name: "SubstringRemoveLast1", runFunction: run_SubstringRemoveLast1, tags: [.validation, .api, .String]),
   BenchmarkInfo(name: "LessSubstringSubstring", runFunction: run_LessSubstringSubstring, tags: [.validation, .api, .String]),
   BenchmarkInfo(name: "LessSubstringSubstringGenericComparable", runFunction: run_LessSubstringSubstringGenericComparable, tags: [.validation, .api, .String]),
   BenchmarkInfo(name: "StringFromLongWholeSubstring", runFunction: run_StringFromLongWholeSubstring, tags: [.validation, .api, .String]),
@@ -34,6 +36,8 @@ let longWide = "fὢasὢodὢijὢadὢolὢsjὢalὢsdὢjlὢasὢdfὢijὢ
 let (s1, ss1) = equivalentWithDistinctBuffers()
 let (s2, ss2) = equivalentWithDistinctBuffers()
 
+let quiteLong = String(repeating: "0", count: 10_000)[...]
+
 @inline(never)
 public func run_SubstringFromLongString(_ N: Int) {
   var s = longWide
@@ -126,6 +130,24 @@ public func run_EqualSubstringSubstringGenericEquatable(_ N: Int) {
   }
 }
 
+@inline(never)
+public func run_SubstringRemoveFirst1(_ N: Int) {
+  for _ in 1...N {
+    var s = quiteLong
+    s.removeFirst(1)
+    blackHole(s.first == "0")
+  }
+}
+
+@inline(never)
+public func run_SubstringRemoveLast1(_ N: Int) {
+  for _ in 1...N {
+    var s = quiteLong
+    s.removeLast(1)
+    blackHole(s.first == "0")
+  }
+}
+
 /*
 func checkEqual<T, U>(_ x: T, _ y: U)
 where T : StringProtocol, U : StringProtocol {

cmake/modules/AddSwift.cmake

@@ -23,7 +23,7 @@ function(_swift_gyb_target_sources target scope)
 
     add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${generated}
       COMMAND
-        $<TARGET_FILE:Python2::Interpreter> ${SWIFT_SOURCE_DIR}/utils/gyb -D CMAKE_SIZEOF_VOID_P=${CMAKE_SIZEOF_VOID_P} ${SWIFT_GYB_FLAGS} -o ${CMAKE_CURRENT_BINARY_DIR}/${generated}.tmp ${absolute}
+        $<TARGET_FILE:Python3::Interpreter> ${SWIFT_SOURCE_DIR}/utils/gyb -D CMAKE_SIZEOF_VOID_P=${CMAKE_SIZEOF_VOID_P} ${SWIFT_GYB_FLAGS} -o ${CMAKE_CURRENT_BINARY_DIR}/${generated}.tmp ${absolute}
       COMMAND
         ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/${generated}.tmp ${CMAKE_CURRENT_BINARY_DIR}/${generated}
       COMMAND

cmake/modules/SwiftHandleGybSources.cmake

@@ -1,8 +1,6 @@
 include(SwiftAddCustomCommandTarget)
 include(SwiftSetIfArchBitness)
 
-find_package(Python2 COMPONENTS Interpreter REQUIRED)
-
 # Create a target to process single gyb source with the 'gyb' tool.
 #
 # handle_gyb_source_single(
@@ -60,7 +58,7 @@ function(handle_gyb_source_single dependency_out_var_name)
       COMMAND
           "${CMAKE_COMMAND}" -E make_directory "${dir}"
       COMMAND
-          "$<TARGET_FILE:Python2::Interpreter>" "${gyb_tool}" ${SWIFT_GYB_FLAGS} ${GYB_SINGLE_FLAGS} -o "${GYB_SINGLE_OUTPUT}.tmp" "${GYB_SINGLE_SOURCE}"
+          "$<TARGET_FILE:Python3::Interpreter>" "${gyb_tool}" ${SWIFT_GYB_FLAGS} ${GYB_SINGLE_FLAGS} -o "${GYB_SINGLE_OUTPUT}.tmp" "${GYB_SINGLE_SOURCE}"
       COMMAND
           "${CMAKE_COMMAND}" -E copy_if_different "${GYB_SINGLE_OUTPUT}.tmp" "${GYB_SINGLE_OUTPUT}"
       COMMAND

docs/Android.md

@@ -103,10 +103,9 @@ $ NDK_PATH="path/to/android-ndk21"
 $ build/Ninja-ReleaseAssert/swift-linux-x86_64/bin/swiftc \     # The Swift compiler built in the previous step.
                                                                 # The location of the tools used to build Android binaries
     -tools-directory ${NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin/ \
-    -target armv7a-none-linux-androideabi \                     # Targeting android-armv7.
-    -sdk ${NDK_PATH}/platforms/android-21/arch-arm \            # Use the same architecture and API version as you used to build the stdlib in the previous step.
-    -Xclang-linker -nostdlib++ \                                # Don't link libc++, and supply the path to libgcc.
+    -target armv7a-none-linux-androideabi \                     # Targeting android-armv7, and supply the path to libgcc.
     -L ${NDK_PATH}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/lib/gcc/arm-linux-androideabi/4.9.x/armv7-a \
+    -sdk ${NDK_PATH}/platforms/android-21/arch-arm \            # Use the same architecture and API version as you used to build the stdlib in the previous step.
     hello.swift
 ```
 

docs/ExternalResources.md

@@ -0,0 +1,176 @@
+# External Resources
+
+The official [Swift blog](https://swift.org/blog/) has a lot of useful
+information, such as [how runtime reflection works][mirror-post] and how the
+compiler's [new diagnostic architecture][diagnostic-arch-post] is structured.
+
+[mirror-post]: https://swift.org/blog/how-mirror-works/
+[diagnostic-arch-post]: https://swift.org/blog/new-diagnostic-arch-overview/
+
+This page lists some external resources apart from the Swift blog which may be
+helpful for people interested in contributing to Swift. The resources are listed
+in reverse-chronological order and organized by topic.
+
+<!--
+Some resources don't fall cleanly into one topic bucket or another; in such a
+case we break the tie arbitrarily.
+-->
+
+<!--
+The textual descriptions should be written in a way that makes it clear
+which topics are discussed, and what a potential contributor to Swift
+will gain from it. This is usually different from the talk's abstract.
+-->
+
+## Contributing Guides and Tips
+
+- [Steps for first PR and asking questions][] by Varun Gandhi (Swift forum
+  comment, Dec 2019).
+- [Contributing to Swift for the First Time][] by Robert Pieta (talk, Jun 2019):
+  This talk describes Robert's experience contributing to
+  `swift-corelibs-foundation` and later `swift`, providing a step-by-step guide
+  for new contributors to get their feet wet.
+- [Contributing to Swift compiler][] by Yusuke Kita (lightning talk, Apr 2019).
+- Contributing to Swift ([Part 1][Contributing to Swift - Part 1],
+  [Part 2][Contributing to Swift - Part 2]) by Suyash Srijan (blog post series,
+  Dec 2018).
+- [Setting up a compile-edit-test cycle][] by Robert Widmann (Swift forum
+  comment, May 2018)
+- [Becoming an Effective Contributor to Swift][] by Harlan Haskins and Robert
+  Widmann (talk, Apr 2018): Covers the following topics:
+  - The organization of different projects part of the Swift monorepo.
+  - How to set up an environment for building Swift.
+  - Common utilities and compiler flags for debugging different stages.
+  - Common build configurations.
+  - Tips and tricks for testing: early exiting, filtering tests,
+    testing Swift and Objective-C together, using `PrettyStackTrace`.
+  - A live demo fixing a crash:
+    - Copying program arguments into Xcode for debugging.
+    - Using `dump()` for print debugging.
+- [Getting Started with Swift Compiler Development][] by Brian Gesiak (blog post
+  series, Aug 2017 - Jun 2018)
+- [Contributing to Open Source Swift][] by Jesse Squires (talk, Mar 2016):
+  Covers the following topics:
+  - The overall compiler pipeline.
+  - The organization of different projects part of the Swift monorepo,
+    including some "difficulty levels" for different projects.
+  - Tips for contributing effectively.
+
+[Steps for first PR and asking questions]:
+https://forums.swift.org/t/getting-started-with-swift-compiler-development/31502/2
+[Contributing to Swift for the First Time]: https://youtu.be/51j7TrFNKiA
+[Contributing to Swift compiler]: https://youtu.be/HAXJsgYniqE
+[Contributing to Swift - Part 1]: https://medium.com/kinandcartacreated/contributing-to-swift-part-1-ea19108a2a54
+[Contributing to Swift - Part 2]:
+https://medium.com/kinandcartacreated/contributing-to-swift-part-2-efebcf7b6c93
+[Setting up a compile-edit-test cycle]: https://forums.swift.org/t/need-a-workflow-advice/12536/14
+[Becoming an Effective Contributor to Swift]: https://youtu.be/oGJKsp-pZPk
+[Getting Started with Swift Compiler Development]: https://modocache.io/getting-started-with-swift-development
+[Contributing to Open Source Swift]: https://youtu.be/Ysa2n8ZX-YY
+
+## AST
+
+- [The secret life of types in Swift][] by Slava Pestov (blog post, Jul 2016):
+  This blog post describes the representation of Swift types inside the compiler.
+  It covers many important concepts: `TypeLoc` vs `TypeRepr` vs `Type`, the
+  representation of generic types, substitutions and more.
+  <!-- TODO: It would be great to integrate some of the descriptions
+       in this blog post into the compiler's own doc comments. -->
+
+[The secret life of types in Swift]: https://medium.com/@slavapestov/the-secret-life-of-types-in-swift-ff83c3c000a5
+
+### libSyntax and SwiftSyntax
+
+- [An overview of SwiftSyntax][] by Luciano Almeida (blog post, Apr 2019):
+  This post provides a quick tour of libSyntax and SwiftSyntax.
+- [Improving Swift Tools with libSyntax][] by Harlan Haskins (talk, Sep 2017):
+  This talk describes the design of libSyntax/SwiftSyntax and discusses some
+  useful APIs. It also describes how to write a simple Swift formatter using the
+  library.
+
+[An overview of SwiftSyntax]: https://medium.com/@lucianoalmeida1/an-overview-of-swiftsyntax-cf1ae6d53494
+[Improving Swift Tools with libSyntax]: https://youtu.be/5ivuYGxW_3M
+
+## Type checking and inference
+
+- [Implementing Swift Generics][] by Slava Pestov and John McCall (talk, Oct 2017):
+  This talk dives into how Swift's compilation scheme for generics balances
+  (a) modularity and separate compilation with (b) the ability to pass unboxed
+  values and (c) the flexibility to optionally generate fully specialized code.
+  It covers the following type-checking related topics: type-checking generic
+  contexts, requirement paths and canonicalized generic signatures.
+- [A Type System from Scratch][] by Robert Widmann (talk, Apr 2017):
+  This talk covers several topics related to type-checking and inference in Swift:
+  - Understanding sequent notation which can be used to represent typing judgments.
+  - An overview of how bidirectional type-checking works.
+  - Examples of checking and inferring types for some Swift expressions.
+  - Interaction complexity of different type system features with type inference.
+  - Type variables and constraint graphs.
+
+[Implementing Swift Generics]: https://youtu.be/ctS8FzqcRug
+[A Type System from Scratch]: https://youtu.be/IbjoA5xVUq0
+
+## SIL
+
+- [Ownership SSA][] by Michael Gottesman (talk, Oct 2019): This talk describes
+  efficiency and correctness challenges with automatic reference counting and
+  how including ownership semantics in the compiler's intermediate representation
+  helps tackles those challenges.
+- [How to talk to your kids about SIL type use][] by Slava Pestov (blog post,
+  Jul 2016): This blog post describes several important SIL concepts: object
+  vs address types, AST -> SIL type lowering, trivial vs loadable vs
+  address-only SIL types, abstraction patterns and more.
+- [Swift's High-Level IR][] by Joe Groff and Chris Lattner (talk, Oct 2015):
+  This talk describes the goals and design of SIL. It covers the following:
+  - Some commonly used SIL instructions and how they are motivated by language
+    features.
+  - Some early passes in SIL processing, such as mandatory inlining,
+    box-to-stack promotion and definite initialization.
+  - Why SIL is useful as an intermediate representation between the AST and
+    LLVM IR.
+
+[Ownership SSA]: https://youtu.be/qy3iZPHZ88o
+[How to talk to your kids about SIL type use]: https://medium.com/@slavapestov/how-to-talk-to-your-kids-about-sil-type-use-6b45f7595f43
+[Swift's High-Level IR]: https://youtu.be/Ntj8ab-5cvE
+
+## Code generation, runtime and ABI
+
+- [How Swift Achieved Dynamic Linking Where Rust Couldn't][] by Alexis
+  Beingessner (blog post, Nov 2019): This blog post describes Swift's approach
+  for compiling polymorphic functions, contrasting it with the strategy used by
+  Rust and C++ (monomorphization). It covers the following topics: ABI stability,
+  library evolution, resilient type layout, reabstraction, materialization,
+  ownership and calling conventions.
+- [arm64e: An ABI for Pointer Authentication][] by Ahmed Bougacha and John McCall
+  (talk, Oct 2019): This talk does not mention Swift specifically, but provides a
+  good background on the arm64e ABI and pointer authentication. The ABI affects
+  parts of IR generation, the Swift runtime, and small parts of the standard
+  library using unsafe code.
+- [Exploiting The Swift ABI][] by Robert Widmann (talk, July 2019):
+  This talk is a whirlwind tour of different aspects of the Swift ABI and runtime
+  touching the following topics: reflection, type layout, ABI entrypoints,
+  Swift's compilation model for generics and archetypes, witness tables,
+  relative references, context descriptors and more.
+- [Efficiently Implementing Runtime Metadata][] by Joe Groff and Doug Gregor
+  (talk, Oct 2018): This talk covers the use of relative references in Swift's
+  runtime metadata structures. After describing some important metrics impacted
+  by the use of dynamic libraries, it goes through the different kinds of
+  relative references used in the Swift runtime and the resulting tooling and
+  performance benefits.
+- [Coroutine Representations and ABIs in LLVM][] by John McCall (talk, Oct 2018):
+  This talk describes several points in the design space for coroutines, diving
+  into important implementation tradeoffs. It explains how different language
+  features can be built on top of coroutines and how they impose different
+  design requirements. It also contrasts C++20's coroutines feature with
+  Swift's accessors, describing the differences between the two as implemented
+  in LLVM.
+- [Implementing Swift Generics][]: This talk is mentioned in the type-checking
+  section. It also covers the following code generation and runtime topics:
+  value witness tables, type metadata, abstraction patterns, reabstraction,
+  reabstraction thunks and protocol witness tables.
+
+[How Swift Achieved Dynamic Linking Where Rust Couldn't]: https://gankra.github.io/blah/swift-abi/
+[arm64e: An ABI for Pointer Authentication]: https://youtu.be/C1nZvpEBfYA
+[Exploiting The Swift ABI]: https://youtu.be/0rHG_Pa86oA
+[Efficiently Implementing Runtime Metadata]: https://youtu.be/G3bpj-4tWVU
+[Coroutine Representations and ABIs in LLVM]: https://youtu.be/wyAbV8AM9PM

docs/README.md

@@ -252,12 +252,10 @@ They are preserved mostly for historical interest.
 
 ## External Resources
 
-The official [Swift blog](https://swift.org/blog/) contains a lot of useful
-information, such as how library evolution works and how the compiler's new
-diagnostic architecture is structured, helping us provide more precise
-diagnostics.
-
-TODO: Add a new document ExternalResources.md.
+External resources are listed in [docs/ExternalResources.md](ExternalResources.md).
+These cover a variety of topics,
+such as the design of different aspects of the Swift compiler and runtime
+and contributing to the project more effectively.
 
 ## Uncategorized
 

include/swift/AST/Decl.h

@@ -5502,35 +5502,10 @@ class ParamDecl : public VarDecl {
   }
 
   /// Does this parameter reject temporary pointer conversions?
-  bool isNonEphemeral() const {
-    if (getAttrs().hasAttribute<NonEphemeralAttr>())
-      return true;
-
-    // Only pointer parameters can be non-ephemeral.
-    auto ty = getInterfaceType();
-    if (!ty->lookThroughSingleOptionalType()->getAnyPointerElementType())
-      return false;
-
-    // Enum element pointer parameters are always non-ephemeral.
-    auto *parentDecl = getDeclContext()->getAsDecl();
-    if (parentDecl && isa<EnumElementDecl>(parentDecl))
-      return true;
-
-    return false;
-  }
+  bool isNonEphemeral() const;
 
   /// Attempt to apply an implicit `@_nonEphemeral` attribute to this parameter.
-  void setNonEphemeralIfPossible() {
-    // Don't apply the attribute if this isn't a pointer param.
-    auto type = getInterfaceType();
-    if (!type->lookThroughSingleOptionalType()->getAnyPointerElementType())
-      return;
-
-    if (!getAttrs().hasAttribute<NonEphemeralAttr>()) {
-      auto &ctx = getASTContext();
-      getAttrs().add(new (ctx) NonEphemeralAttr(/*IsImplicit*/ true));
-    }
-  }
+  void setNonEphemeralIfPossible();
 
   /// Remove the type of this varargs element designator, without the array
   /// type wrapping it.  A parameter like "Int..." will have formal parameter

include/swift/AST/DiagnosticsSIL.def

@@ -496,12 +496,6 @@ NOTE(autodiff_cannot_differentiate_through_multiple_results,none,
      "cannot differentiate through multiple results", ())
 NOTE(autodiff_cannot_differentiate_through_inout_arguments,none,
      "cannot differentiate through 'inout' arguments", ())
-// TODO(TF-1149): Remove this diagnostic.
-NOTE(autodiff_loadable_value_addressonly_tangent_unsupported,none,
-     "cannot yet differentiate value whose type %0 has a compile-time known "
-     "size, but whose 'TangentVector' contains stored properties of unknown "
-     "size; consider modifying %1 to use fewer generic parameters in stored "
-     "properties", (Type, Type))
 NOTE(autodiff_enums_unsupported,none,
      "differentiating enum values is not yet supported", ())
 NOTE(autodiff_stored_property_parent_not_differentiable,none,

include/swift/AST/DiagnosticsSema.def

@@ -1075,6 +1075,9 @@ ERROR(optional_base_not_unwrapped,none,
 NOTE(optional_base_chain,none,
      "chain the optional using '?' to access member %0 only for non-'nil' "
      "base values", (DeclNameRef))
+NOTE(optional_base_remove_optional_for_keypath_root, none,
+     "use unwrapped type %0 as key path root", (Type))
+
 ERROR(missing_unwrap_optional_try,none,
       "value of optional type %0 not unwrapped; did you mean to use 'try!' "
       "or chain with '?'?",
@@ -2689,7 +2692,8 @@ ERROR(decl_from_hidden_module,none,
       "in an extension with public or '@usableFromInline' members|"
       "in an extension with conditional conformances}2; "
       "%select{%3 has been imported as implementation-only|"
-      "it is an SPI imported from %3}4",
+      "it is an SPI imported from %3|"
+      "it is SPI}4",
       (DescriptiveDeclKind, DeclName, unsigned, Identifier, unsigned))
 ERROR(conformance_from_implementation_only_module,none,
       "cannot use conformance of %0 to %1 %select{here|as property wrapper here|"
@@ -4769,7 +4773,8 @@ WARNING(resilience_decl_unavailable_warn,
 ERROR(inlinable_decl_ref_from_hidden_module,
       none, "%0 %1 cannot be used in " FRAGILE_FUNC_KIND "2 "
       "because %select{%3 was imported implementation-only|"
-      "it is an SPI imported from %3}4",
+      "it is an SPI imported from %3|"
+      "it is SPI}4",
       (DescriptiveDeclKind, DeclName, unsigned, Identifier, unsigned))
 
 #undef FRAGILE_FUNC_KIND

include/swift/AST/Expr.h

@@ -5380,6 +5380,13 @@ class KeyPathExpr : public Expr {
       return Loc;
     }
 
+    SourceRange getSourceRange() const {
+      if (auto *indexExpr = getIndexExpr()) {
+        return indexExpr->getSourceRange();
+      }
+      return Loc;
+    }
+    
     Kind getKind() const {
       return KindValue;
     }

include/swift/AST/SILOptions.h

@@ -41,7 +41,7 @@ class SILOptions {
   int UnrollThreshold = 250;
 
   /// The number of threads for multi-threaded code generation.
-  int NumThreads = 0;
+  unsigned NumThreads = 0;
 
   /// Controls whether to pull in SIL from partial modules during the
   /// merge modules step. Could perhaps be merged with the link mode