Skip to content

Commit 7dc9191

Browse files
author
marcrasi
authored
inmemoryfrontend (#26848)
1 parent fc4c701 commit 7dc9191

File tree

10 files changed

+298
-0
lines changed

10 files changed

+298
-0
lines changed

‎include/swift/Subsystems.h

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,23 @@ namespace swift {
284284
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
285285
const SILModule *M = nullptr);
286286

287+
// SWIFT_ENABLE_TENSORFLOW
288+
/// Serializes a module or single source file to a memory buffer, and returns
289+
/// the memory buffer in an output parameter. Does not write to the
290+
/// filesystem.
291+
///
292+
/// \param moduleBuffer will be set to a pointer to the serialized module
293+
/// buffer. nullptr is allowed, in which case the module
294+
/// will not be serialized.
295+
/// \param moduleDocBuffer will be set to a pointer to the serialized module
296+
/// doc buffer. nullptr is allowed, in which case the
297+
/// module doc will not be serialized.
298+
void serializeToMemory(ModuleOrSourceFile DC,
299+
const SerializationOptions &options,
300+
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
301+
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
302+
const SILModule *M = nullptr);
303+
287304
/// Get the CPU, subtarget feature options, and triple to use when emitting code.
288305
std::tuple<llvm::TargetOptions, std::string, std::vector<std::string>,
289306
std::string>

‎lib/Serialization/Serialization.cpp

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5170,6 +5170,30 @@ void swift::serializeToBuffers(
51705170
}
51715171
}
51725172

5173+
// SWIFT_ENABLE_TENSORFLOW
5174+
void swift::serializeToMemory(
5175+
ModuleOrSourceFile DC, const SerializationOptions &options,
5176+
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
5177+
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer, const SILModule *M) {
5178+
if (moduleBuffer) {
5179+
SharedTimer timer("Serialization, swiftmodule, to memory");
5180+
llvm::SmallString<1024> buf;
5181+
llvm::raw_svector_ostream stream(buf);
5182+
Serializer::writeToStream(stream, DC, M, options);
5183+
*moduleBuffer =
5184+
llvm::make_unique<llvm::SmallVectorMemoryBuffer>(std::move(buf));
5185+
}
5186+
5187+
if (moduleDocBuffer) {
5188+
SharedTimer timer("Serialization, swiftdoc, to memory");
5189+
llvm::SmallString<1024> buf;
5190+
llvm::raw_svector_ostream stream(buf);
5191+
writeDocToStream(stream, DC, options.GroupInfoPath);
5192+
*moduleDocBuffer =
5193+
llvm::make_unique<llvm::SmallVectorMemoryBuffer>(std::move(buf));
5194+
}
5195+
}
5196+
51735197
void swift::serialize(ModuleOrSourceFile DC,
51745198
const SerializationOptions &options,
51755199
const SILModule *M) {

‎tools/CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,6 @@ if(SWIFT_HOST_VARIANT STREQUAL "macosx")
5858
endif()
5959

6060
add_swift_tool_subdirectory(swift-reflection-dump)
61+
62+
# SWIFT_ENABLE_TENSORFLOW
63+
add_swift_tool_subdirectory(libInMemoryFrontend)

‎tools/libInMemoryFrontend/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
include_directories(
2+
${CMAKE_CURRENT_SOURCE_DIR}/include
3+
)
4+
5+
add_subdirectory(lib)

‎tools/libInMemoryFrontend/include/libInMemoryFrontend/InMemoryFrontend.h

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
//===--- InMemoryFrontend.h - Frontend operations, in memory ----*- C++ -*-===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2019 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifndef SWIFT_LIBINMEMORYFRONTEND_INMEMORYFRONTEND_H
14+
#define SWIFT_LIBINMEMORYFRONTEND_INMEMORYFRONTEND_H
15+
16+
#include "swift/Frontend/Frontend.h"
17+
#include "llvm/Support/MemoryBuffer.h"
18+
19+
namespace swift {
20+
namespace inmemoryfrontend {
21+
22+
/// Given a fully setup CompilerInstance, configured to emit one module, runs
23+
/// the compilation and emits the module to a memory buffer, without writing to
24+
/// the filesystem. Emits error information to the CompilerInstance's
25+
/// DiagnosticEngine.
26+
///
27+
/// \param moduleBuffer will be set to a pointer to the serialized module
28+
/// buffer. nullptr is allowed, in which case the module
29+
/// will not be serialized.
30+
/// \param moduleDocBuffer will be set to a pointer to the serialized module
31+
/// doc buffer. nullptr is allowed, in which case the
32+
/// module doc will not be serialized.
33+
/// \return true on error.
34+
bool compileSwiftModule(CompilerInstance &CI,
35+
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
36+
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer);
37+
38+
} // end namespace inmemoryfrontend
39+
} // end namespace swift
40+
41+
#endif

‎tools/libInMemoryFrontend/lib/CMakeLists.txt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
add_swift_host_library(libInMemoryFrontend STATIC
2+
InMemoryFrontend.cpp
3+
)
4+
5+
target_link_libraries(libInMemoryFrontend PRIVATE
6+
swiftFrontend
7+
swiftSerialization
8+
swiftSIL
9+
)

‎tools/libInMemoryFrontend/lib/InMemoryFrontend.cpp

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
//===--- InMemoryFrontend.cpp - Frontend operations, in memory --*- C++ -*-===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2019 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#include "libInMemoryFrontend/InMemoryFrontend.h"
14+
15+
#include "swift/SIL/SILModule.h"
16+
#include "swift/Subsystems.h"
17+
18+
namespace swift {
19+
namespace inmemoryfrontend {
20+
21+
bool compileSwiftModule(CompilerInstance &CI,
22+
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
23+
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer) {
24+
CI.performSema();
25+
if (CI.getDiags().hadAnyError())
26+
return true;
27+
28+
auto SILMod = performSILGeneration(CI.getMainModule(), CI.getSILOptions());
29+
if (!SILMod)
30+
return true;
31+
32+
SerializationOptions SerOpts;
33+
SILMod->setSerializeSILAction([&]() {
34+
serializeToMemory(CI.getMainModule(), SerOpts, moduleBuffer,
35+
moduleDocBuffer, SILMod.get());
36+
});
37+
38+
if (CI.performSILProcessing(SILMod.get()))
39+
return true;
40+
41+
return false;
42+
}
43+
44+
} // end namespace inmemoryfrontend
45+
} // end namespace swift

‎unittests/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,7 @@ if(SWIFT_INCLUDE_TOOLS)
2828
if(SWIFT_BUILD_SOURCEKIT)
2929
add_subdirectory(SourceKit)
3030
endif()
31+
32+
add_subdirectory(libInMemoryFrontend)
3133
endif()
3234

‎unittests/libInMemoryFrontend/CMakeLists.txt

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
add_swift_unittest(libInMemoryFrontendTests
2+
InMemoryFrontendTests.cpp
3+
)
4+
5+
target_link_libraries(libInMemoryFrontendTests PRIVATE
6+
libInMemoryFrontend
7+
swiftDriver
8+
)
9+
10+
target_compile_definitions(libInMemoryFrontendTests PRIVATE
11+
SWIFTLIB_DIR=\"${SWIFTLIB_DIR}\"
12+
)
13+
14+
include_directories(
15+
${SWIFT_SOURCE_DIR}/tools/libInMemoryFrontend/include
16+
)

‎unittests/libInMemoryFrontend/InMemoryFrontendTests.cpp

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
#include "libInMemoryFrontend/InMemoryFrontend.h"
2+
#include "swift/AST/DiagnosticConsumer.h"
3+
#include "swift/Driver/FrontendUtil.h"
4+
#include "swift/Frontend/Frontend.h"
5+
#include "swift/Serialization/ModuleFile.h"
6+
#include "llvm/ADT/IntrusiveRefCntPtr.h"
7+
#include "llvm/Support/FileSystem.h"
8+
#include "llvm/Support/raw_ostream.h"
9+
#include "gtest/gtest.h"
10+
11+
using namespace swift;
12+
13+
class StreamDiagConsumer : public DiagnosticConsumer {
14+
llvm::raw_ostream &OS;
15+
16+
public:
17+
StreamDiagConsumer(llvm::raw_ostream &OS) : OS(OS) {}
18+
19+
void
20+
handleDiagnostic(SourceManager &SM, SourceLoc Loc, DiagnosticKind Kind,
21+
StringRef FormatString,
22+
ArrayRef<DiagnosticArgument> FormatArgs,
23+
const DiagnosticInfo &Info,
24+
const SourceLoc bufferIndirectlyCausingDiagnostic) override {
25+
switch (Kind) {
26+
case DiagnosticKind::Error:
27+
OS << "error: ";
28+
break;
29+
case DiagnosticKind::Warning:
30+
OS << "warning: ";
31+
break;
32+
case DiagnosticKind::Note:
33+
OS << "note: ";
34+
break;
35+
case DiagnosticKind::Remark:
36+
OS << "remark: ";
37+
break;
38+
}
39+
DiagnosticEngine::formatDiagnosticText(OS, FormatString, FormatArgs);
40+
}
41+
};
42+
43+
static StringRef getRuntimeLibPath() {
44+
return llvm::sys::path::parent_path(SWIFTLIB_DIR);
45+
}
46+
47+
class InMemoryFrontendTest : public ::testing::Test {
48+
protected:
49+
InMemoryFrontendTest()
50+
: MemFS(new llvm::vfs::InMemoryFileSystem()),
51+
FS(new llvm::vfs::OverlayFileSystem(llvm::vfs::getRealFileSystem())),
52+
ErrOS(ErrStr), DiagConsumer(ErrOS) {
53+
FS->pushOverlay(MemFS);
54+
55+
CI.addDiagnosticConsumer(&DiagConsumer);
56+
CI.getSourceMgr().setFileSystem(FS);
57+
}
58+
59+
bool ParseArgsAndSetupInstance(llvm::ArrayRef<const char *> OrigArgs) {
60+
SmallVector<const char *, 16> Args;
61+
Args.push_back("-resource-dir");
62+
Args.push_back(getRuntimeLibPath().data());
63+
Args.append(OrigArgs.begin(), OrigArgs.end());
64+
65+
// Without this configuration option, the clang tries to emit object files
66+
// for the modules that it compiles. To do this, it looks up the current
67+
// triple in the llvm TargetRegistry. We have not initialized the
68+
// TargetRegistry, so it fails.
69+
Invocation.getClangImporterOptions().DetailedPreprocessingRecord = true;
70+
71+
bool ParseResult = driver::getSingleFrontendInvocationFromDriverArguments(
72+
Args, CI.getDiags(), [&](ArrayRef<const char *> FrontendArgs) {
73+
return Invocation.parseArgs(FrontendArgs, CI.getDiags());
74+
});
75+
if (ParseResult)
76+
return true;
77+
78+
return CI.setup(Invocation);
79+
}
80+
81+
llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> MemFS;
82+
llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> FS;
83+
84+
SmallString<32> ErrStr;
85+
llvm::raw_svector_ostream ErrOS;
86+
StreamDiagConsumer DiagConsumer;
87+
88+
CompilerInstance CI;
89+
CompilerInvocation Invocation;
90+
};
91+
92+
TEST_F(InMemoryFrontendTest, SemaError) {
93+
MemFS->addFile("/file1.swift", /*ModificationTime=*/0,
94+
llvm::MemoryBuffer::getMemBuffer("let x: String = \"hello\"",
95+
"/file1.swift"));
96+
MemFS->addFile(
97+
"/file2.swift", /*ModificationTime=*/0,
98+
llvm::MemoryBuffer::getMemBuffer("let y: Int = x", "/file2.swift"));
99+
100+
const char *Args[] = {"/file1.swift", "/file2.swift"};
101+
bool SetupResult = ParseArgsAndSetupInstance(Args);
102+
ASSERT_FALSE(SetupResult) << ErrStr;
103+
104+
std::unique_ptr<llvm::MemoryBuffer> ModBuf;
105+
std::unique_ptr<llvm::MemoryBuffer> ModDocBuf;
106+
bool CompileResult =
107+
inmemoryfrontend::compileSwiftModule(CI, &ModBuf, &ModDocBuf);
108+
EXPECT_TRUE(CompileResult);
109+
EXPECT_EQ("error: cannot convert value of type 'String' to specified type "
110+
"'Int'",
111+
ErrStr);
112+
}
113+
114+
TEST_F(InMemoryFrontendTest, Success) {
115+
MemFS->addFile("/file1.swift", /*ModificationTime=*/0,
116+
llvm::MemoryBuffer::getMemBuffer("let x: String = \"hello\"",
117+
"/file1.swift"));
118+
MemFS->addFile(
119+
"/file2.swift", /*ModificationTime=*/0,
120+
llvm::MemoryBuffer::getMemBuffer("let y: String = x", "/file2.swift"));
121+
122+
const char *Args[] = {"/file1.swift", "/file2.swift"};
123+
bool SetupResult = ParseArgsAndSetupInstance(Args);
124+
ASSERT_FALSE(SetupResult) << ErrStr;
125+
126+
std::unique_ptr<llvm::MemoryBuffer> ModBuf;
127+
std::unique_ptr<llvm::MemoryBuffer> ModDocBuf;
128+
bool CompileResult =
129+
inmemoryfrontend::compileSwiftModule(CI, &ModBuf, &ModDocBuf);
130+
ASSERT_FALSE(CompileResult) << ErrStr;
131+
ASSERT_TRUE(ModBuf);
132+
ASSERT_TRUE(ModDocBuf);
133+
134+
EXPECT_EQ(serialization::Status::Valid,
135+
serialization::validateSerializedAST(ModBuf->getBuffer()).status);
136+
}

0 commit comments

Comments
 (0)