Skip to content

Commit a6714bc

Browse files
cachemeifyoucancachemeifyoucan
authored
Merge pull request #66887 from cachemeifyoucan/eng/PR-cas-option-updates
Update CAS/Caching options as final candidate.
2 parents da8e84c + 284eb2a commit a6714bc

34 files changed

+427
-131
lines changed

‎include/swift/AST/DiagnosticsFrontend.def

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,6 @@ REMARK(replay_output, none, "replay output file '%0': key '%1'", (StringRef, Str
494494
REMARK(output_cache_miss, none, "cache miss output file '%0': key '%1'", (StringRef, StringRef))
495495

496496
// CAS related diagnostics
497-
ERROR(error_create_cas, none, "failed to create CAS '%0' (%1)", (StringRef, StringRef))
498497
ERROR(error_invalid_cas_id, none, "invalid CASID '%0' (%1)", (StringRef, StringRef))
499498
ERROR(error_cas, none, "CAS error encountered: %0", (StringRef))
500499

‎include/swift/AST/ModuleDependencies.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
#include "swift/Basic/LLVM.h"
2222
#include "swift/AST/Import.h"
23+
#include "clang/CAS/CASOptions.h"
2324
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
2425
#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
2526
#include "llvm/ADT/ArrayRef.h"
@@ -767,6 +768,9 @@ class SwiftDependencyScanningService {
767768
ModuleDependenciesKindMap ModuleDependenciesMap;
768769
};
769770

771+
/// The CASOption created the Scanning Service if used.
772+
llvm::Optional<clang::CASOptions> CASOpts;
773+
770774
/// The persistent Clang dependency scanner service
771775
Optional<clang::tooling::dependencies::DependencyScanningService>
772776
ClangScanningService;
@@ -863,7 +867,7 @@ class SwiftDependencyScanningService {
863867
void overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance);
864868

865869
/// Setup caching service.
866-
void setupCachingDependencyScanningService(CompilerInstance &Instance);
870+
bool setupCachingDependencyScanningService(CompilerInstance &Instance);
867871
private:
868872
/// Enforce clients not being allowed to query this cache directly, it must be
869873
/// wrapped in an instance of `ModuleDependenciesCache`.

‎include/swift/Basic/LangOptions.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "swift/Basic/LLVM.h"
2424
#include "swift/Basic/Version.h"
2525
#include "swift/Config.h"
26+
#include "clang/CAS/CASOptions.h"
2627
#include "llvm/ADT/ArrayRef.h"
2728
#include "llvm/ADT/Hashing.h"
2829
#include "llvm/ADT/SmallString.h"
@@ -820,7 +821,7 @@ namespace swift {
820821
std::string Optimization;
821822

822823
/// clang CASOptions.
823-
std::string CASPath;
824+
llvm::Optional<clang::CASOptions> CASOpts;
824825

825826
/// Cache key for imported bridging header.
826827
std::string BridgingHeaderPCHCacheKey;

‎include/swift/Frontend/CachingUtils.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ bool replayCachedCompilerOutputs(
3939
llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache,
4040
llvm::cas::ObjectRef BaseKey, DiagnosticEngine &Diag,
4141
const FrontendInputsAndOutputs &InputsAndOutputs,
42-
CachingDiagnosticsProcessor &CDP);
42+
CachingDiagnosticsProcessor &CDP, bool CacheRemarks);
4343

4444
/// Load the cached compile result from cache.
4545
std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(

‎include/swift/Frontend/FrontendOptions.h

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "llvm/ADT/Hashing.h"
2222
#include "llvm/ADT/Optional.h"
2323
#include "llvm/ADT/StringMap.h"
24+
#include "clang/CAS/CASOptions.h"
2425

2526
#include <string>
2627
#include <vector>
@@ -122,11 +123,17 @@ class FrontendOptions {
122123
/// The module for which we should verify all of the generic signatures.
123124
std::string VerifyGenericSignaturesInModule;
124125

125-
/// Use CAS.
126-
bool EnableCAS = false;
126+
/// Enable compiler caching.
127+
bool EnableCaching = false;
127128

128-
/// The CAS Path.
129-
std::string CASPath;
129+
/// Enable compiler caching remarks.
130+
bool EnableCachingRemarks = false;
131+
132+
/// Skip replaying outputs from cache.
133+
bool CacheSkipReplay = false;
134+
135+
/// CASOptions
136+
clang::CASOptions CASOpts;
130137

131138
/// CASFS Root.
132139
std::vector<std::string> CASFSRootIDs;

‎include/swift/Option/FrontendOptions.td

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1217,6 +1217,23 @@ def always_compile_output_files :
12171217
Flag<["-"], "always-compile-output-files">,
12181218
HelpText<"Always compile output files even it might not change the results">;
12191219

1220+
// CAS/Caching related options.
1221+
def allow_unstable_cache_key_for_testing: Flag<["-"], "allow-unstable-cache-key-for-testing">,
1222+
HelpText<"Allow compilation caching with unstable inputs for testing purpose">;
1223+
1224+
def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">,
1225+
HelpText<"Cache Key for bridging header pch">;
1226+
1227+
def clang_include_tree: Flag<["-"], "clang-include-tree">,
1228+
HelpText<"Use clang include tree">;
1229+
1230+
def cas_fs: Separate<["-"], "cas-fs">,
1231+
HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"<cas-id>">;
1232+
1233+
def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">,
1234+
HelpText<"Clang Include Tree CASID">, MetaVarName<"<cas-id>">;
1235+
1236+
12201237
def experimental_spi_only_imports :
12211238
Flag<["-"], "experimental-spi-only-imports">,
12221239
HelpText<"Enable use of @_spiOnly imports">;

‎include/swift/Option/Options.td

Lines changed: 18 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1812,36 +1812,32 @@ def gcc_toolchain: Separate<["-"], "gcc-toolchain">,
18121812
MetaVarName<"<path>">,
18131813
HelpText<"Specify a directory where the clang importer and clang linker can find headers and libraries">;
18141814

1815-
def cas_path: Separate<["-"], "cas-path">,
1815+
def cache_compile_job: Flag<["-"], "cache-compile-job">,
18161816
Flags<[FrontendOption, NewDriverOnlyOption]>,
1817-
HelpText<"Path to CAS">, MetaVarName<"<path>">;
1817+
HelpText<"Enable compiler caching">;
18181818

1819-
def allow_unstable_cache_key_for_testing: Flag<["-"], "allow-unstable-cache-key-for-testing">,
1820-
Flags<[FrontendOption, HelpHidden, NoDriverOption]>,
1821-
HelpText<"Allow compilation caching with unstable inputs for testing purpose">;
1822-
1823-
def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">,
1824-
Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
1825-
HelpText<"Cache Key for bridging header pch">;
1819+
def cache_remarks: Flag<["-"], "cache-remarks">,
1820+
Flags<[FrontendOption, NewDriverOnlyOption]>,
1821+
HelpText<"Show remarks for compiler caching">;
18261822

1827-
def clang_include_tree: Flag<["-"], "clang-include-tree">,
1828-
Flags<[FrontendOption, NoDriverOption]>,
1829-
HelpText<"Use clang include tree">;
1823+
def cache_disable_replay: Flag<["-"], "cache-disable-replay">,
1824+
Flags<[FrontendOption, NewDriverOnlyOption]>,
1825+
HelpText<"Skip loading the compilation result from cache">;
18301826

1827+
def cas_path: Separate<["-"], "cas-path">,
1828+
Flags<[FrontendOption, NewDriverOnlyOption]>,
1829+
HelpText<"Path to CAS">, MetaVarName<"<path>">;
18311830

1832-
// END ONLY SUPPORTED IN NEW DRIVER
1831+
def cas_plugin_path: Separate<["-"], "cas-plugin-path">,
1832+
Flags<[FrontendOption, NewDriverOnlyOption]>,
1833+
HelpText<"Path to CAS Plugin">, MetaVarName<"<path>">;
18331834

1834-
def enable_cas: Flag<["-"], "enable-cas">,
1835-
Flags<[FrontendOption, NoDriverOption]>,
1836-
HelpText<"Enable CAS for swift-frontend">;
1835+
def cas_plugin_option: Separate<["-"], "cas-plugin-option">,
1836+
Flags<[FrontendOption, NewDriverOnlyOption]>,
1837+
HelpText<"Option pass to CAS Plugin">, MetaVarName<"<name>=<option>">;
18371838

1838-
def cas_fs: Separate<["-"], "cas-fs">,
1839-
Flags<[FrontendOption, NoDriverOption]>,
1840-
HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"<cas-id>">;
18411839

1842-
def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">,
1843-
Flags<[FrontendOption, NoDriverOption]>,
1844-
HelpText<"Clang Include Tree CASID">, MetaVarName<"<cas-id>">;
1840+
// END ONLY SUPPORTED IN NEW DRIVER
18451841

18461842

18471843
def plugin_search_Group : OptionGroup<"<plugin search options>">;

‎lib/AST/ModuleDependencies.cpp

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -417,12 +417,25 @@ void SwiftDependencyScanningService::overlaySharedFilesystemCacheForCompilation(
417417
Instance.getSourceMgr().setFileSystem(depFS);
418418
}
419419

420-
void SwiftDependencyScanningService::setupCachingDependencyScanningService(
420+
bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
421421
CompilerInstance &Instance) {
422-
if (!Instance.getInvocation().getFrontendOptions().EnableCAS)
423-
return;
422+
if (!Instance.getInvocation().getFrontendOptions().EnableCaching)
423+
return false;
424+
425+
if (CASOpts) {
426+
// If CASOption matches, the service is initialized already.
427+
if (*CASOpts == Instance.getInvocation().getFrontendOptions().CASOpts)
428+
return false;
429+
430+
// CASOption mismatch, return error.
431+
Instance.getDiags().diagnose(
432+
SourceLoc(), diag::error_cas,
433+
"conflicting CAS options used in scanning service");
434+
return true;
435+
}
424436

425437
// Setup CAS.
438+
CASOpts = Instance.getInvocation().getFrontendOptions().CASOpts;
426439
CAS = Instance.getSharedCASInstance();
427440

428441
// Add SDKSetting file.
@@ -454,17 +467,12 @@ void SwiftDependencyScanningService::setupCachingDependencyScanningService(
454467
auto CachingFS =
455468
llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore());
456469
if (!CachingFS) {
457-
Instance.getDiags().diagnose(SourceLoc(), diag::error_create_cas,
458-
"CachingOnDiskFS",
470+
Instance.getDiags().diagnose(SourceLoc(), diag::error_cas,
459471
toString(CachingFS.takeError()));
460-
return;
472+
return true;
461473
}
462474
CacheFS = std::move(*CachingFS);
463475

464-
clang::CASOptions CASOpts;
465-
CASOpts.CASPath = Instance.getInvocation().getFrontendOptions().CASPath;
466-
CASOpts.ensurePersistentCAS();
467-
468476
UseClangIncludeTree =
469477
Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree;
470478
const clang::tooling::dependencies::ScanningOutputFormat ClangScanningFormat =
@@ -474,10 +482,13 @@ void SwiftDependencyScanningService::setupCachingDependencyScanningService(
474482

475483
ClangScanningService.emplace(
476484
clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
477-
ClangScanningFormat, CASOpts, Instance.getSharedCASInstance(),
478-
Instance.getSharedCacheInstance(),
485+
ClangScanningFormat,
486+
Instance.getInvocation().getFrontendOptions().CASOpts,
487+
Instance.getSharedCASInstance(), Instance.getSharedCacheInstance(),
479488
UseClangIncludeTree ? nullptr : CacheFS,
480489
/* ReuseFileManager */ false, /* OptimizeArgs */ false);
490+
491+
return false;
481492
}
482493

483494
SwiftDependencyScanningService::ContextSpecificGlobalCacheState *
@@ -611,8 +622,8 @@ Optional<const ModuleDependencyInfo *> ModuleDependenciesCache::findDependency(
611622
scannerContextHash);
612623
// During a scan, only produce the cached source module info for the current
613624
// module under scan.
614-
if (optionalDep.hasValue()) {
615-
auto dep = optionalDep.getValue();
625+
if (optionalDep) {
626+
auto dep = *optionalDep;
616627
if (dep->getAsSwiftSourceModule() &&
617628
moduleName != mainScanModuleName &&
618629
moduleName != "DummyMainModuleForResolvingCrossImportOverlays") {

‎lib/ClangImporter/ClangImporter.cpp

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -709,13 +709,28 @@ importer::getNormalInvocationArguments(
709709
llvm::sys::path::get_separator() +
710710
"apinotes").str());
711711

712-
if (!importerOpts.CASPath.empty()) {
713-
invocationArgStrs.push_back("-Xclang");
714-
invocationArgStrs.push_back("-fcas-path");
715-
invocationArgStrs.push_back("-Xclang");
716-
invocationArgStrs.push_back(importerOpts.CASPath);
712+
if (importerOpts.CASOpts) {
717713
invocationArgStrs.push_back("-Xclang");
718714
invocationArgStrs.push_back("-fno-pch-timestamp");
715+
if (!importerOpts.CASOpts->CASPath.empty()) {
716+
invocationArgStrs.push_back("-Xclang");
717+
invocationArgStrs.push_back("-fcas-path");
718+
invocationArgStrs.push_back("-Xclang");
719+
invocationArgStrs.push_back(importerOpts.CASOpts->CASPath);
720+
}
721+
if (!importerOpts.CASOpts->PluginPath.empty()) {
722+
invocationArgStrs.push_back("-Xclang");
723+
invocationArgStrs.push_back("-fcas-plugin-path");
724+
invocationArgStrs.push_back("-Xclang");
725+
invocationArgStrs.push_back(importerOpts.CASOpts->PluginPath);
726+
for (auto Opt : importerOpts.CASOpts->PluginOptions) {
727+
invocationArgStrs.push_back("-Xclang");
728+
invocationArgStrs.push_back("-fcas-plugin-option");
729+
invocationArgStrs.push_back("-Xclang");
730+
invocationArgStrs.push_back(
731+
(llvm::Twine(Opt.first) + "=" + Opt.second).str());
732+
}
733+
}
719734
}
720735
}
721736

‎lib/ClangImporter/ClangModuleDependencyScanner.cpp

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -238,10 +238,21 @@ void ClangImporter::recordModuleDependencies(
238238
std::string IncludeTree =
239239
clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : "";
240240

241-
if (!RootID.empty() || !IncludeTree.empty()) {
241+
if (ctx.ClangImporterOpts.CASOpts) {
242242
swiftArgs.push_back("-enable-cas");
243-
swiftArgs.push_back("-cas-path");
244-
swiftArgs.push_back(ctx.ClangImporterOpts.CASPath);
243+
if (!ctx.ClangImporterOpts.CASOpts->CASPath.empty()) {
244+
swiftArgs.push_back("-cas-path");
245+
swiftArgs.push_back(ctx.ClangImporterOpts.CASOpts->CASPath);
246+
}
247+
if (!ctx.ClangImporterOpts.CASOpts->PluginPath.empty()) {
248+
swiftArgs.push_back("-cas-plugin-path");
249+
swiftArgs.push_back(ctx.ClangImporterOpts.CASOpts->PluginPath);
250+
for (auto Opt : ctx.ClangImporterOpts.CASOpts->PluginOptions) {
251+
swiftArgs.push_back("-cas-plugin-option");
252+
swiftArgs.push_back(
253+
(llvm::Twine(Opt.first) + "=" + Opt.second).str());
254+
}
255+
}
245256
}
246257

247258
if (!RootID.empty()) {
@@ -334,10 +345,20 @@ void ClangImporter::recordBridgingHeaderOptions(
334345

335346
llvm::for_each(clangArgs, addClangArg);
336347

337-
if (!ctx.ClangImporterOpts.CASPath.empty()) {
348+
if (ctx.ClangImporterOpts.CASOpts) {
338349
swiftArgs.push_back("-enable-cas");
339-
swiftArgs.push_back("-cas-path");
340-
swiftArgs.push_back(ctx.ClangImporterOpts.CASPath);
350+
if (!ctx.ClangImporterOpts.CASOpts->CASPath.empty()) {
351+
swiftArgs.push_back("-cas-path");
352+
swiftArgs.push_back(ctx.ClangImporterOpts.CASOpts->CASPath);
353+
}
354+
if (!ctx.ClangImporterOpts.CASOpts->PluginPath.empty()) {
355+
swiftArgs.push_back("-cas-plugin-path");
356+
swiftArgs.push_back(ctx.ClangImporterOpts.CASOpts->PluginPath);
357+
for (auto Opt : ctx.ClangImporterOpts.CASOpts->PluginOptions) {
358+
swiftArgs.push_back("-cas-plugin-option");
359+
swiftArgs.push_back((llvm::Twine(Opt.first) + "=" + Opt.second).str());
360+
}
361+
}
341362
}
342363

343364
if (auto Tree = deps.IncludeTreeID) {
@@ -387,12 +408,12 @@ Optional<const ModuleDependencyInfo*> ClangImporter::getModuleDependencies(
387408
std::vector<std::string> commandLineArgs =
388409
getClangDepScanningInvocationArguments(ctx);
389410
auto optionalWorkingDir = computeClangWorkingDirectory(commandLineArgs, ctx);
390-
if (!optionalWorkingDir.hasValue()) {
411+
if (!optionalWorkingDir) {
391412
ctx.Diags.diagnose(SourceLoc(), diag::clang_dependency_scan_error,
392413
"Missing '-working-directory' argument");
393414
return None;
394415
}
395-
std::string workingDir = optionalWorkingDir.getValue();
416+
std::string workingDir = *optionalWorkingDir;
396417

397418
auto moduleCachePath = getModuleCachePathFromClang(getClangInstance());
398419
auto lookupModuleOutput =
@@ -452,12 +473,12 @@ bool ClangImporter::addBridgingHeaderDependencies(
452473
std::vector<std::string> commandLineArgs =
453474
getClangDepScanningInvocationArguments(ctx, StringRef(bridgingHeader));
454475
auto optionalWorkingDir = computeClangWorkingDirectory(commandLineArgs, ctx);
455-
if (!optionalWorkingDir.hasValue()) {
476+
if (!optionalWorkingDir) {
456477
ctx.Diags.diagnose(SourceLoc(), diag::clang_dependency_scan_error,
457478
"Missing '-working-directory' argument");
458479
return true;
459480
}
460-
std::string workingDir = optionalWorkingDir.getValue();
481+
std::string workingDir = *optionalWorkingDir;
461482

462483
auto moduleCachePath = getModuleCachePathFromClang(getClangInstance());
463484
auto lookupModuleOutput =

‎lib/DependencyScan/DependencyScanningTool.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,8 @@ DependencyScanningTool::initCompilerInstanceForScan(
279279
}
280280

281281
// Setup the caching service after the instance finishes setup.
282-
ScanningService->setupCachingDependencyScanningService(*Instance);
282+
if (ScanningService->setupCachingDependencyScanningService(*Instance))
283+
return std::make_error_code(std::errc::invalid_argument);
283284

284285
(void)Instance->getMainModule();
285286

‎lib/DependencyScan/ScanDependencies.cpp

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,7 @@ static llvm::Error resolveExplicitModuleInputs(
421421
dependencyInfoCopy.updateCommandLine(commandLine);
422422

423423
// Handle CAS options.
424-
if (instance.getInvocation().getFrontendOptions().EnableCAS) {
424+
if (instance.getInvocation().getFrontendOptions().EnableCaching) {
425425
// Merge CASFS from clang dependency.
426426
auto &CASFS = cache.getScanService().getSharedCachingFS();
427427
auto &CAS = CASFS.getCAS();
@@ -1764,7 +1764,9 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) {
17641764
deserializeDependencyCache(instance, service);
17651765
// Wrap the filesystem with a caching `DependencyScanningWorkerFilesystem`
17661766
service.overlaySharedFilesystemCacheForCompilation(instance);
1767-
service.setupCachingDependencyScanningService(instance);
1767+
if (service.setupCachingDependencyScanningService(instance))
1768+
return true;
1769+
17681770
ModuleDependenciesCache cache(service,
17691771
instance.getMainModule()->getNameStr().str(),
17701772
instance.getInvocation().getModuleScanningHash());
@@ -1832,7 +1834,9 @@ bool swift::dependencies::batchScanDependencies(
18321834

18331835
SwiftDependencyScanningService singleUseService;
18341836
singleUseService.overlaySharedFilesystemCacheForCompilation(instance);
1835-
singleUseService.setupCachingDependencyScanningService(instance);
1837+
if (singleUseService.setupCachingDependencyScanningService(instance))
1838+
return true;
1839+
18361840
ModuleDependenciesCache cache(singleUseService,
18371841
instance.getMainModule()->getNameStr().str(),
18381842
instance.getInvocation().getModuleScanningHash());

0 commit comments

Comments
 (0)