Skip to content

Add libQuote to apple/swift:tensorflow CI #26708

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Add libQuote to apple/swift:tensorflow CI #26708

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c7b2dd7
Build libQuote together with the S4TF toolchain
Aug 17, 2019
c8ffbe2
Rename references to libQuote in various scripts
Aug 17, 2019
184f38a
Address feedback
Aug 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,8 @@ Below is more information about TensorFlow-related build arguments.
* Default value: None.
* `tensorflow-swift-apis`: A path to the [tensorflow/swift-apis](https://github.com/tensorflow/swift-apis) deep learning library repository.
* Default value: `tensorflow-swift-apis` if the [tensorflow/swift-apis](https://github.com/tensorflow/swift-apis) repository is cloned. Otherwise, none.
* `tensorflow-swift-quote`: A path to the [Quote](https://github.com/tensorflow/swift) library repository.
* Default value: `tensorflow-swift-quote` if the [Quote](https://github.com/tensorflow/swift) repository is cloned. Otherwise, none.

### Build systems

Expand Down
6 changes: 6 additions & 0 deletions stdlib/public/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ if(SWIFT_BUILD_STDLIB AND SWIFT_ENABLE_TENSORFLOW)
add_subdirectory(TensorFlow)
endif()

# SWIFT_ENABLE_TENSORFLOW
if(SWIFT_BUILD_STDLIB AND SWIFT_ENABLE_TENSORFLOW)
add_subdirectory(Quote)
endif()
# SWIFT_ENABLE_TENSORFLOW END

if(SWIFT_BUILD_STDLIB OR SWIFT_BUILD_REMOTE_MIRROR)
add_subdirectory(Reflection)
add_subdirectory(SwiftRemoteMirror)
Expand Down
52 changes: 52 additions & 0 deletions stdlib/public/Quote/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#===-------------- CMakeLists.txt - Build the Quote library ---------------===#
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
#===----------------------------------------------------------------------===#
#
# SWIFT_ENABLE_TENSORFLOW
#
#===----------------------------------------------------------------------===#

if(NOT SWIFT_ENABLE_TENSORFLOW)
return()
endif()

set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
set(swift_stdlib_compile_flags "${SWIFT_RUNTIME_SWIFT_COMPILE_FLAGS}")

set(SOURCES "")

# Copy Quote sources, if they exist.
if (TENSORFLOW_SWIFT_QUOTE)
file(GLOB_RECURSE TENSORFLOW_SWIFT_QUOTE_SOURCES
"${TENSORFLOW_SWIFT_QUOTE}/Sources/Quote/*.swift")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is really an anti-pattern for CMake. If you do this, subsequent incremental builds will be incorrect as this will not pick up new files being added - it is evaluated once. It is much better to explicitly list out the sources.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the right way of doing this? My knowledge of CMake is cursory at best, so I'd appreciate any help :)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is to explicitly list out all the source files manually. Doing so allows CMake to indicate to the build tool which files to track. You would literally just enumerate the files and write them out.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a note: the file list here actually refers to files from a different repository (a different build-script product).

In practice, this means that when files are added/removed, we'll need to update the CMake source list when we update utils/update_checkout/update_checkout_config.json.

list(APPEND SOURCES "${TENSORFLOW_SWIFT_QUOTE_SOURCES}")
endif()

add_swift_target_library(swiftQuote ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
"${SOURCES}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BTW, once you actually list out the files, you can just delete the SOURCES variable and inline the full list of files.


TARGET_SDKS OSX LINUX
PRIVATE_LINK_LIBRARIES "${TF_LIBRARIES}"
SWIFT_MODULE_DEPENDS SwiftOnoneSupport
SWIFT_MODULE_DEPENDS_IOS Darwin
SWIFT_MODULE_DEPENDS_OSX Darwin
SWIFT_MODULE_DEPENDS_TVOS Darwin
SWIFT_MODULE_DEPENDS_WATCHOS Darwin
SWIFT_MODULE_DEPENDS_LINUX Glibc
SWIFT_MODULE_DEPENDS_FREEBSD Glibc
SWIFT_MODULE_DEPENDS_CYGWIN Glibc
SWIFT_MODULE_DEPENDS_HAIKU Glibc
SWIFT_COMPILE_FLAGS "${swift_stdlib_compile_flags}"
LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
INSTALL_IN_COMPONENT stdlib
# NOTE: This is a workaround for https://github.com/apple/swift/pull/24382,
# which changed the default install_name_dir to `/usr/bin/swift`.
DARWIN_INSTALL_NAME_DIR "@rpath")
7 changes: 7 additions & 0 deletions utils/build-script
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,13 @@ class BuildScriptInvocation(object):
"--tensorflow-swift-apis (was '{}')"
.format(args.tensorflow_swift_apis))

if args.tensorflow_swift_quote is not None:
if not os.path.isdir(args.tensorflow_swift_quote):
diagnostics.fatal(
"invalid repository specified for "
"--tensorflow-swift-quote (was '{}')"
.format(args.tensorflow_swift_quote))

if args.enable_tensorflow_gpu:
os.environ['TF_NEED_CUDA'] = '1'
if '--config=cuda' not in args.tensorflow_bazel_options:
Expand Down
17 changes: 17 additions & 0 deletions utils/build-script-impl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ KNOWN_SETTINGS=(
tensorflow-target-include-dir "" "Path to target Tensorflow headers"
tensorflow-target-lib-dir "" "Path to target TensorFlow libraries"
tensorflow-swift-apis "" "Path to TensorFlow deep learning library repository"
tensorflow-swift-quote "" "Path to Quote library repository"
)

# Centralized access point for traced command invocation.
Expand Down Expand Up @@ -1235,6 +1236,7 @@ PLAYGROUNDSUPPORT_SOURCE_DIR="${WORKSPACE}/swift-xcode-playground-support"
# SWIFT_ENABLE_TENSORFLOW
TENSORFLOW_SOURCE_DIR="${WORKSPACE}/tensorflow"
TENSORFLOW_SWIFT_APIS_DIR="${WORKSPACE}/tensorflow-swift-apis"
TENSORFLOW_SWIFT_QUOTE_DIR="${WORKSPACE}/tensorflow-swift-quote"

if [[ ! "${SKIP_BUILD_PLAYGROUNDSUPPORT}" && ! -d ${PLAYGROUNDSUPPORT_SOURCE_DIR} ]]; then
echo "Couldn't find PlaygroundSupport source directory."
Expand All @@ -1250,6 +1252,10 @@ if [[ "${ENABLE_TENSORFLOW}" ]]; then
echo "Invalid TensorFlow high-level APIs repository specified."
exit 1
fi
if [[ "${TENSORFLOW_SWIFT_QUOTE}" && ! -d "${TENSORFLOW_SWIFT_QUOTE}" ]]; then
echo "Invalid Quote repository specified."
exit 1
fi
fi

# Symlink clang into the llvm tree.
Expand Down Expand Up @@ -2620,6 +2626,17 @@ for host in "${ALL_HOSTS[@]}"; do
-DTENSORFLOW_SWIFT_APIS:PATH="${TENSORFLOW_SWIFT_APIS}"
)
fi

# Handle Quote repository.
if [[ ! "${TENSORFLOW_SWIFT_QUOTE}" && -d "${TENSORFLOW_SWIFT_QUOTE_DIR}" ]] ; then
TENSORFLOW_SWIFT_QUOTE="${TENSORFLOW_SWIFT_QUOTE_DIR}"
fi
if [[ "${TENSORFLOW_SWIFT_QUOTE}" ]] ; then
cmake_options=(
"${cmake_options[@]}"
-DTENSORFLOW_SWIFT_QUOTE:PATH="${TENSORFLOW_SWIFT_QUOTE}"
)
fi
fi

if contains_product "lldb" ; then
Expand Down
3 changes: 3 additions & 0 deletions utils/build_swift/driver_arguments.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1063,6 +1063,9 @@ def create_argument_parser():
option('--tensorflow-swift-apis', store_path,
default=None,
help='Path to a TensorFlow deep learning library repository.')
option('--tensorflow-swift-quote', store_path,
default=None,
help='Path to a Quote repository.')
option('--tensorflow-bazel-options', append,
type=argparse.ShellSplitType(),
default=[],
Expand Down
2 changes: 2 additions & 0 deletions utils/build_swift/tests/expected_options.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@
'tensorflow_target_lib_dir': None,
'tensorflow_target_include_dir': None,
'tensorflow_swift_apis': None,
'tensorflow_swift_quote': None,
'host_bazel': None,
'tensorflow_bazel_options': [],
}
Expand Down Expand Up @@ -618,6 +619,7 @@ class IgnoreOption(_BaseOption):
PathOption('--tensorflow-target-lib-dir'),
PathOption('--tensorflow-target-include-dir'),
PathOption('--tensorflow-swift-apis'),
PathOption('--tensorflow-swift-quote'),
PathOption('--host-bazel'),
AppendOption('--tensorflow-bazel-options'),
]
3 changes: 3 additions & 0 deletions utils/update_checkout/update-checkout-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
"remote": { "id": "tensorflow/tensorflow" } },
"tensorflow-swift-apis": {
"remote": { "id": "tensorflow/swift-apis" } },
"tensorflow-swift-quote": {
"remote": { "id": "tensorflow/swift" } },
"icu": {
"remote": { "id": "unicode-org/icu" },
"platforms": [ "Linux" ]
Expand Down Expand Up @@ -486,6 +488,7 @@
"libcxx": "swift-DEVELOPMENT-SNAPSHOT-2019-08-12-a",
"tensorflow": "c6719f20911f8aa6b6d9cded1c0f7761cb9c69a0",
"tensorflow-swift-apis": "3629ddc49b706df89f89fa3e92e495e6e3f42332",
"tensorflow-swift-quote": "03a61b10adfff4d08c23ed487b0a0a786932a07d",
"indexstore-db": "swift-DEVELOPMENT-SNAPSHOT-2019-08-12-a",
"sourcekit-lsp": "swift-DEVELOPMENT-SNAPSHOT-2019-08-12-a"
}
Expand Down