Merge github/tensorflow into copy of github/master (using imerge)

Author: Dave Abrahams

.gitignore

@@ -18,6 +18,7 @@
 # vim swap files
 .*.sw[a-z]
 .sw?
+*.vscode
 
 #==============================================================================#
 # Explicit files to ignore (only matches one).

CMakeLists.txt

@@ -86,6 +86,12 @@ option(SWIFT_BUILD_STDLIB_EXTRA_TOOLCHAIN_CONTENT
     "If not building stdlib, controls whether to build 'stdlib/toolchain' content"
     TRUE)
 
+# `SWIFT_ENABLE_TENSOFRFLOW` affects lit tests, adding "tensorflow" as an
+# available feature.
+option(SWIFT_ENABLE_TENSORFLOW
+    "Enable TensorFlow in the compiler"
+    FALSE)
+
 # In many cases, the CMake build system needs to determine whether to include
 # a directory, or perform other actions, based on whether the stdlib or SDK is
 # being built at all -- statically or dynamically. Please note that these
@@ -373,6 +379,19 @@ option(SWIFT_DISABLE_OBJC_INTEROP
     "Disable Objective-C interoperability even on platforms what would normally have it"
     FALSE)
 
+#
+# User-configurable TensorFlow specific options.
+#
+set(SWIFT_TENSORFLOW_HOST_LIB_DIR "" CACHE PATH
+    "If set, directory with TensorFlow host libraries to be linked into swiftc.")
+set(SWIFT_TENSORFLOW_TARGET_LIB_DIR "" CACHE PATH
+    "If set, directory with TensorFlow target libraries to be linked into swift programs.")
+
+set(SWIFT_TENSORFLOW_HOST_INCLUDE_DIR "" CACHE PATH
+    "If set, directory with host TensorFlow headers.")
+set(SWIFT_TENSORFLOW_TARGET_INCLUDE_DIR "" CACHE PATH
+    "If set, directory with target TensorFlow headers.")
+
 #
 # User-configurable experimental options.  Do not use in production builds.
 #
@@ -1084,6 +1103,11 @@ if(SWIFT_BUILD_SYNTAXPARSERLIB OR SWIFT_BUILD_SOURCEKIT)
   endif()
 endif()
 
+# Enable TensorFlow in compiler code.
+if(SWIFT_ENABLE_TENSORFLOW)
+  add_definitions(-DSWIFT_ENABLE_TENSORFLOW)
+endif()
+
 # Add all of the subdirectories, where we actually do work.
 
 ###############

CONTRIBUTING.md

@@ -5,7 +5,23 @@ license](https://swift.org/LICENSE.txt).
 
 ---
 
-Before submitting the pull request, please make sure you have [tested your
-changes](https://github.com/apple/swift/blob/master/docs/ContinuousIntegration.md)
-and that they follow the Swift project [guidelines for contributing
-code](https://swift.org/contributing/#contributing-code).
+Before submitting a pull request, please make sure that your changes follow the
+[Swift project guidelines for contributing code](https://swift.org/contributing/#contributing-code).
+
+We request that code changes unrelated to Swift for TensorFlow be submitted to
+the [upstream Swift repository](https://github.com/apple/swift). For example,
+code formatting changes that do not affect Swift for TensorFlow source code
+should be submitted upstream.
+
+It is a good idea to discuss any non-trivial submissions with the project
+maintainers before submitting a pull request: please join the
+[[email protected]](https://groups.google.com/a/tensorflow.org/d/forum/swift)
+mailing list to participate in discussions.
+
+All changes to existing Swift source code should be marked clearly with a
+`SWIFT_ENABLE_TENSORFLOW` comment at the top of every diff hunk. This makes
+it easier to merge from upstream.
+
+Continuous integration (CI) is still being set up, so if you submit a pull
+request with non-trivial changes that require test coverage, please anticipate
+some delay before we can properly review and merge it.

README.md

@@ -1,7 +1,9 @@
-<img src="https://swift.org/assets/images/swift.svg" alt="Swift logo" height="70" >
-
-# Swift Programming Language
+# Swift for TensorFlow
 
+| OS | CI platform | x86_64 | GPU |
+|---|:---:|:---:|:---:|
+| **macOS** | Google Kokoro | ![Build Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/macos-swift-tf-release.svg) | - |
+| **Ubuntu 16.04** | Swift.org CI | [![Build Status](https://ci-external.swift.org/job/oss-swift-RA-linux-ubuntu-16.04-tensorflow/lastCompletedBuild/badge/icon)](https://ci-external.swift.org/job/oss-swift-RA-linux-ubuntu-16.04-tensorflow) | [![Build Status](https://ci-external.swift.org/job/oss-swift-RA-linux-ubuntu-16.04-tensorflow-gpu/lastCompletedBuild/badge/icon)](https://ci-external.swift.org/job/oss-swift-RA-linux-ubuntu-16.04-tensorflow-gpu) |
 
 | | **Architecture** | **Master** | **Package** |
 |---|:---:|:---:|:---:|
@@ -34,6 +36,13 @@
 |**[macOS 10.13](https://github.com/apple/swift-community-hosted-continuous-integration/blob/master/nodes/x86_64_macos_high_sierra_tensorflow.json)** | x86_64 |[![Build Status](https://ci-external.swift.org/job/oss-swift-RA-macOS-tensorflow/lastCompletedBuild/badge/icon)](https://ci-external.swift.org/job/oss-swift-RA-macOS-tensorflow)|
 |**[Ubuntu 16.04 (GPU)](https://github.com/apple/swift-community-hosted-continuous-integration/blob/master/nodes/x86_64_ubuntu_16_04_tensorflow_gpu.json)** | x86_64 |[![Build Status](https://ci-external.swift.org/job/oss-swift-RA-linux-ubuntu-16.04-tensorflow-gpu/lastCompletedBuild/badge/icon)](https://ci-external.swift.org/job/oss-swift-RA-linux-ubuntu-16.04-tensorflow-gpu)|
 
+<!-- SWIFT_ENABLE_TENSORFLOW -->
+
+Swift for TensorFlow is a new programming language for TensorFlow. It is a copy of the compiler for the [Swift Programming Language](https://swift.org) that adds first-class compiler and language support for machine learning.
+
+This repository covers the compiler and standard libraries. Please visit the [documentation repository](https://github.com/tensorflow/swift) for more information about the project, including a project overview, technical details, and guidelines for contributing. To use Swift for TensorFlow out of the box, follow the [installation instructions](https://github.com/tensorflow/swift/blob/master/Installation.md). To build from source, follow the instructions below.
+<!-- SWIFT_ENABLE_TENSORFLOW END -->
+
 ## Welcome to Swift
 
 Swift is a high-performance system programming language.  It has a clean
@@ -82,37 +91,36 @@ If you are interested in:
 
 [Getting Started guide]: /docs/HowToGuides/GettingStarted.md
 
-### Swift Toolchains
+### Swift For TensorFlow Toolchains
 
 #### Building
 
-Swift toolchains are created using the script
-[build-toolchain](https://github.com/apple/swift/blob/master/utils/build-toolchain). This
-script is used by swift.org's CI to produce snapshots and can allow for one to
+Swift for TensorFlow toolchains are created using the script
+[build-toolchain-tensorflow](https://github.com/apple/swift/blob/tensorflow/utils/build-toolchain-tensorflow).
+This script is used by swift.org's CI to produce snapshots and can allow for one to
 locally reproduce such builds for development or distribution purposes. A typical 
 invocation looks like the following:
 
 ```
-  $ ./swift/utils/build-toolchain $BUNDLE_PREFIX
+  $ ./swift/utils/build-toolchain-tensorflow $BUNDLE_PREFIX
 ```
 
-where ``$BUNDLE_PREFIX`` is a string that will be prepended to the build 
-date to give the bundle identifier of the toolchain's ``Info.plist``. For 
-instance, if ``$BUNDLE_PREFIX`` was ``com.example``, the toolchain 
-produced will have the bundle identifier ``com.example.YYYYMMDD``. It 
-will be created in the directory you run the script with a filename 
-of the form: ``swift-LOCAL-YYYY-MM-DD-a-osx.tar.gz``.
+where ``$BUNDLE_PREFIX`` is a string that will be prepended to the build
+date to give the bundle identifier of the toolchain's ``Info.plist``. For
+instance, if ``$BUNDLE_PREFIX`` was ``com.example``, the toolchain
+produced will have the bundle identifier ``com.example.YYYYMMDD``. It
+will be created in the directory you run the script with a filename
+ of the form: ``swift-tensorflow-LOCAL-YYYY-MM-DD-a-osx.tar.gz``.
 
-Beyond building the toolchain, ``build-toolchain`` also supports the 
-following (non-exhaustive) set of useful options::
+Beyond building the toolchain, ``build-toolchain-tensorflow`` also supports the
+following (non-exhaustive) set of useful options:
 
 - ``--dry-run``: Perform a dry run build. This is off by default.
 - ``--test``: Test the toolchain after it has been compiled. This is off by default.
-- ``--distcc``: Use distcc to speed up the build by distributing the c++ part of
-  the swift build. This is off by default.
+- ``--pkg`` (macOS only): Build a toolchain installer package (`.pkg`). This is off by default.
 
 More options may be added over time. Please pass ``--help`` to
-``build-toolchain`` to see the full set of options.
+``build-toolchain-tensorflow`` to see the full set of options.
 
 #### Installing into Xcode
 

cmake/modules/FindTensorFlow.cmake

@@ -0,0 +1,23 @@
+# SWIFT_ENABLE_TENSORFLOW
+# Find TensorFlow.
+
+include(FindPackageHandleStandardArgs)
+
+find_path(TF_INCLUDE_DIR
+  NAMES third_party/tensorflow/c tensorflow/c
+  HINTS ${SWIFT_TENSORFLOW_TARGET_INCLUDE_DIR} /usr/include /usr/local/include)
+if (EXISTS ${TF_INCLUDE_DIR}/third_party/tensorflow/c/c_api.h)
+  # This is experimental and not covered by CI.
+  set(TF_PATH_ADJUSTMENT "third_party")
+else()
+  # Note: This is the normal workflow.
+  set(TF_PATH_ADJUSTMENT "")
+endif()
+
+find_library(TF_LIBRARY
+  NAMES tensorflow
+  HINTS ${SWIFT_TENSORFLOW_TARGET_LIB_DIR} /usr/lib /usr/local/lib)
+set(TF_LIBRARIES ${TF_LIBRARY})
+
+find_package_handle_standard_args(TensorFlow DEFAULT_MSG TF_INCLUDE_DIR TF_LIBRARIES)
+mark_as_advanced(${TF_INCLUDE_DIR} ${TF_LIBRARIES})

docs/ContinuousIntegration.md

@@ -28,7 +28,21 @@ This page is designed to assist in the understanding of proper practices for tes
 
 In order for the Swift project to be able to advance quickly, it is important that we maintain a green build [[1]](#footnote-1). In order to help maintain this green build, the Swift project heavily uses pull request (PR) testing. Specifically, an important general rule is that **all** non-trivial checkins to any Swift Project repository should at least perform a [smoke test](#smoke-testing) if simulators will not be impacted *or* a full [validation test](#validation-testing) if simulators may be impacted. If in addition one is attempting to make a source breaking change across multiple repositories, one should follow the cross repo source breaking changes workflow. We now continue by describing the Swift system for Pull Request testing, @swift-ci:
 
-### @swift-ci
+### @swift-ci (Swift for TensorFlow)
+
+
+Platform     | Comment
+------------ | -------
+All supported platforms     | @swift-ci Please test tensorflow
+All supported platforms     | @swift-ci Please clean test tensorflow
+Linux                       | @swift-ci Please test tensorflow Linux
+Linux GPU                   | @swift-ci Please test tensorflow Linux GPU
+macOS                       | @swift-ci Please test tensorflow macOS
+
+Status can be checked at https://ci-external.swift.org/view/Pull%20Request/.
+
+
+### @swift-ci (general Swift)
 
 Users with [commit access](https://swift.org/contributing/#commit-access) can trigger pull request testing by writing a comment on a PR addressed to the GitHub user @swift-ci. Different tests will run depending on the specific comment used. The current test types are:
 

include/swift/AST/ASTContext.h

@@ -296,7 +296,7 @@ class ASTContext final {
 
   /// Cached mapping from types to their associated tangent spaces.
   llvm::DenseMap<Type, Optional<TangentSpace>> AutoDiffTangentSpaces;
-
+  
   /// A cache of derivative function types per configuration.
   llvm::DenseMap<SILAutoDiffDerivativeFunctionKey, CanSILFunctionType>
       SILAutoDiffDerivativeFunctions;
@@ -309,8 +309,10 @@ class ASTContext final {
   /// Cache of `@derivative` attributes keyed by parameter indices and
   /// derivative function kind. Used to diagnose duplicate `@derivative`
   /// attributes for the same key.
-  // TODO(TF-1042): remove `DerivativeAttrs` from `ASTContext`. Serialize
-  // derivative function configurations per original `AbstractFunctionDecl`.
+  // NOTE(TF-680): relaxing the uniqueness condition to use derivative generic
+  // signature as a key is possible. It requires derivative generic signature
+  // mangling to avoid name collisions for SIL derivative functions with the
+  // same parameter indices but different derivative generic signatures.
   llvm::DenseMap<
       std::tuple<Decl *, IndexSubset *, AutoDiffDerivativeFunctionKind>,
       llvm::SmallPtrSet<DerivativeAttr *, 1>>
@@ -336,6 +338,7 @@ class ASTContext final {
   /// Cache of module names that fail the 'canImport' test in this context.
   llvm::SmallPtrSet<Identifier, 8> FailedModuleImportNames;
 
+public:
   /// Retrieve the allocator for the given arena.
   llvm::BumpPtrAllocator &
   getAllocator(AllocationArena arena = AllocationArena::Permanent) const;
@@ -485,6 +488,35 @@ class ASTContext final {
   /// Retrieve the type Swift.AnyObject.
   CanType getAnyObjectType() const;
 
+  // SWIFT_ENABLE_TENSORFLOW
+  /// Retrieve the decl for TensorFlow.TensorHandle iff the TensorFlow module
+  /// has been imported.  Otherwise, this returns null.
+  ClassDecl *getTensorHandleDecl() const;
+
+  /// Retrieve the decl for TensorFlow.TensorShape iff the TensorFlow module
+  /// has been imported.  Otherwise, this returns null.
+  StructDecl *getTensorShapeDecl() const;
+
+  /// Retrieve the decl for TensorFlow.TensorDataType iff the TensorFlow module
+  /// has been imported.  Otherwise, this returns null.
+  StructDecl *getTensorDataTypeDecl() const;
+
+  /// Retrieve the decl for the Quote module iff it has been imported.
+  /// Otherwise, this returns null.
+  ModuleDecl *getQuoteModule() const;
+
+  /// Retrieve the decl for Quote.Tree iff the Quote module has been imported.
+  /// Otherwise, this returns null.
+  ProtocolDecl *getTreeDecl() const;
+
+  /// Retrieve the decl for Quote.Quote iff the Quote module has been imported.
+  /// Otherwise, this returns null.
+  ClassDecl *getQuoteDecl() const;
+
+  /// Retrieve the decl for Quote.FunctionQuoteN iff the Quote module has been
+  /// imported. Otherwise, this returns null.
+  ClassDecl *getFunctionQuoteDecl(unsigned n) const;
+
   /// Retrieve the type Swift.Never.
   CanType getNeverType() const;
 

include/swift/AST/ASTMangler.h

@@ -342,7 +342,7 @@ class ASTMangler : public Mangler {
   /// \returns \c true if a generic signature was appended, \c false
   /// if it was empty.
   bool appendGenericSignature(GenericSignature sig,
-                              GenericSignature contextSig = nullptr);
+                              GenericSignature contextSig = GenericSignature());
 
   void appendRequirement(const Requirement &reqt);
 

include/swift/AST/Attr.def

@@ -566,6 +566,13 @@ SIMPLE_DECL_ATTR(asyncHandler, AsyncHandler,
   ABIStableToAdd | ABIStableToRemove | APIStableToAdd | APIStableToRemove,
   101)
 
+// SWIFT_ENABLE_TENSORFLOW
+SIMPLE_DECL_ATTR(compilerEvaluable, CompilerEvaluable,
+  OnAccessor | OnFunc | OnConstructor | OnSubscript |
+  ABIStableToAdd | ABIStableToRemove | APIStableToAdd | APIStableToRemove |
+  NotSerialized, 102)
+// SWIFT_ENABLE_TENSORFLOW END
+
 #undef TYPE_ATTR
 #undef DECL_ATTR_ALIAS
 #undef CONTEXTUAL_DECL_ATTR_ALIAS

include/swift/AST/Attr.h

@@ -51,6 +51,7 @@ class Decl;
 class AbstractFunctionDecl;
 class FuncDecl;
 class ClassDecl;
+class FuncDecl;
 class GenericFunctionType;
 class LazyConformanceLoader;
 class LazyMemberLoader;

include/swift/AST/Decl.h

@@ -3502,6 +3502,20 @@ class NominalTypeDecl : public GenericTypeDecl, public IterableDeclContext {
   /// Retrieve information about this type as a property wrapper.
   PropertyWrapperTypeInfo getPropertyWrapperTypeInfo() const;
 
+private:
+  /// Predicate used to filter StoredPropertyRange.
+  struct ToStoredProperty {
+    ToStoredProperty(bool skipInaccessible = false) :
+        skipUserInaccessible(skipInaccessible) {}
+    bool skipUserInaccessible;
+    Optional<VarDecl *> operator()(Decl *decl) const;
+  };
+  
+public:
+  /// A range for iterating the stored member variables of a structure.
+  using StoredPropertyRange = OptionalTransformRange<DeclRange,
+                                                     ToStoredProperty>;
+
   /// Return a collection of the stored member variables of this type.
   ArrayRef<VarDecl *> getStoredProperties() const;
 
@@ -4198,6 +4212,15 @@ enum class KnownDerivableProtocolKind : uint8_t {
   Decodable,
   AdditiveArithmetic,
   Differentiable,
+  // SWIFT_ENABLE_TENSORFLOW
+  PointwiseMultiplicative,
+  ElementaryFunctions,
+  KeyPathIterable,
+  TensorArrayProtocol,
+  TensorGroup,
+  VectorProtocol,
+  EuclideanDifferentiable,
+  // SWIFT_ENABLE_TENSORFLOW END
 };
 
 /// ProtocolDecl - A declaration of a protocol, for example:
@@ -6586,8 +6609,9 @@ AbstractStorageDecl::AccessorRecord::getAccessor(AccessorKind kind) const {
 
 /// This represents a 'case' declaration in an 'enum', which may declare
 /// one or more individual comma-separated EnumElementDecls.
-class EnumCaseDecl final : public Decl,
-    private llvm::TrailingObjects<EnumCaseDecl, EnumElementDecl *> {
+class EnumCaseDecl final
+    : public Decl,
+      private llvm::TrailingObjects<EnumCaseDecl, EnumElementDecl *> {
   friend TrailingObjects;
   friend class Decl;
   SourceLoc CaseLoc;

include/swift/AST/DiagnosticsDriver.def

@@ -190,5 +190,12 @@ WARNING(warn_drv_darwin_sdk_invalid_settings, none,
     "SDK settings were ignored because 'SDKSettings.json' could not be parsed",
     ())
 
+// SWIFT_ENABLE_TENSORFLOW
+ERROR(error_tensorflow_toolchain_repl_not_supported, none,
+      "The Swift for TensorFlow toolchain does not support the Swift REPL. Colab "
+      "(https://github.com/tensorflow/swift/blob/master/Usage.md#colaboratory) and Swift-Jupyter "
+      "(https://github.com/google/swift-jupyter) are supported alternatives.",
+      ())
+
 #define UNDEFINE_DIAGNOSTIC_MACROS
 #include "DefineDiagnosticMacros.h"

include/swift/AST/DiagnosticsParse.def

@@ -1765,5 +1765,22 @@ ERROR(expected_closure_literal,none,
 ERROR(expected_multiple_closures_block_rbrace,none,
       "expected '}' at the end of a trailing closures block", ())
 
+// SWIFT_ENABLE_TENSORFLOW
+ERROR(sil_const_expected_int_datatype,PointsToFirstBadToken,
+      "expected integer datatype ('i[0-9]+', e.g. 'i32')", ())
+ERROR(sil_const_expected_int_value,PointsToFirstBadToken,
+      "expected integer value in SIL constant value", ())
+ERROR(sil_const_expected_fp_datatype,PointsToFirstBadToken,
+      "expected floating point datatype ('f32' or 'f64')", ())
+ERROR(sil_const_expected_fp_value,PointsToFirstBadToken,
+      "expected floating point value in SIL constant value", ())
+ERROR(sil_const_array_expected_rsquare,PointsToFirstBadToken,
+      "expected ']' at end of array 'SymbolicValue'", ())
+ERROR(sil_const_aggregate_expected_rparen,PointsToFirstBadToken,
+      "expected ')' at end of aggregate 'SymbolicValue'", ())
+ERROR(sil_const_expected_fn_sub_conv,PointsToFirstBadToken,
+      "expected '(N)' or '(W)' function substitution convention", ())
+// SWIFT_ENABLE_TENSORFLOW END
+
 #define UNDEFINE_DIAGNOSTIC_MACROS
 #include "DefineDiagnosticMacros.h"