Skip to content

Commit ca749de

Browse files
cachemeifyoucancachemeifyoucan
committed
[Dependency Scanning] Support loading scanner cache state for caching
Teach scanner cache loader to validate the CAS contents when validating dependency graph loaded.
1 parent c1063d0 commit ca749de

File tree

7 files changed

+99
-41
lines changed

7 files changed

+99
-41
lines changed

‎include/swift/AST/DiagnosticsCommon.def

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -232,8 +232,14 @@ REMARK(remark_scanner_invalidate_upstream, none,
232232
"Incremental module scan: Dependency info for module '%0' invalidated due to an out-of-date"
233233
" dependency.", (StringRef))
234234

235-
REMARK(remark_cache_reuse_disabled_with_cas, none,
236-
"Incremental module scan: Re-using serialized module scanning dependency cache disabled in Caching build", ())
235+
REMARK(remark_scanner_invalidate_configuration, none,
236+
"Incremental module scan: Dependency info for module '%0' invalidated due to wrong configuration.", (StringRef))
237+
238+
REMARK(remark_scanner_invalidate_cas_error, none,
239+
"Incremental module scan: Dependency info for module '%0' invalidated due to cas error: %1", (StringRef, StringRef))
240+
241+
REMARK(remark_scanner_invalidate_missing_cas, none,
242+
"Incremental module scan: Dependency info for module '%0' invalidated due to missing CAS input '%1'.", (StringRef, StringRef))
237243

238244
//------------------------------------------------------------------------------
239245
// MARK: custom attribute diagnostics

‎include/swift/AST/ModuleDependencies.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1117,6 +1117,8 @@ class SwiftDependencyScanningService {
11171117
return Mapper->mapToString(Path);
11181118
}
11191119

1120+
bool hasCAS() const { return (bool)CAS; }
1121+
11201122
/// Setup caching service.
11211123
bool setupCachingDependencyScanningService(CompilerInstance &Instance);
11221124

‎include/swift/DependencyScan/ScanDependencies.h

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,24 @@
1414
#define SWIFT_DEPENDENCY_SCANDEPENDENCIES_H
1515

1616
#include "swift-c/DependencyScan/DependencyScan.h"
17-
#include "llvm/ADT/StringRef.h"
18-
#include "llvm/ADT/StringSet.h"
19-
#include "llvm/Support/Error.h"
17+
#include "swift/AST/DiagnosticEngine.h"
18+
#include "llvm/ADT/StringMap.h"
19+
#include "llvm/Support/Chrono.h"
20+
#include "llvm/Support/ErrorOr.h"
21+
#include <unordered_set>
2022

2123
namespace llvm {
2224
class StringSaver;
23-
}
25+
namespace vfs {
26+
class FileSystem;
27+
} // namespace vfs
28+
} // namespace llvm
2429

2530
namespace swift {
2631

2732
class CompilerInvocation;
2833
class CompilerInstance;
34+
class DiagnosticEngine;
2935
class ModuleDependenciesCache;
3036
struct ModuleDependencyID;
3137
struct ModuleDependencyIDHash;

‎lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1819,6 +1819,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
18191819
.CASBridgingHeaderIncludeTreeRootID);
18201820
addIdentifier(swiftTextDeps->moduleCacheKey);
18211821
addIdentifier(swiftTextDeps->userModuleVersion);
1822+
addIdentifier(swiftTextDeps->moduleCacheKey);
18221823
break;
18231824
}
18241825
case swift::ModuleDependencyKind::SwiftBinary: {
@@ -1831,6 +1832,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
18311832
addIdentifier(swiftBinDeps->headerImport);
18321833
addIdentifier(swiftBinDeps->definingModuleInterfacePath);
18331834
addIdentifier(swiftBinDeps->userModuleVersion);
1835+
addIdentifier(swiftBinDeps->moduleCacheKey);
18341836
addStringArray(moduleID,
18351837
ModuleIdentifierArrayKind::HeaderInputModuleDependencies,
18361838
clangHeaderDependencyNames);
@@ -1872,6 +1874,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
18721874
swiftSourceDeps->textualModuleDetails.CASFileSystemRootID);
18731875
addIdentifier(swiftSourceDeps->chainedBridgingHeaderPath);
18741876
addIdentifier(swiftSourceDeps->chainedBridgingHeaderContent);
1877+
addIdentifier(swiftSourceDeps->moduleCacheKey);
18751878
break;
18761879
}
18771880
case swift::ModuleDependencyKind::Clang: {

‎lib/DependencyScan/ScanDependencies.cpp

Lines changed: 58 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
//===----------------------------------------------------------------------===//
1212

1313
#include "swift-c/DependencyScan/DependencyScan.h"
14+
#include "swift/AST/DiagnosticsCommon.h"
1415
#include "swift/Basic/PrettyStackTrace.h"
1516

1617
#include "swift/AST/ASTContext.h"
@@ -1273,32 +1274,27 @@ swift::dependencies::performModuleScan(
12731274
// Load the dependency cache if -reuse-dependency-scan-cache
12741275
// is specified
12751276
if (opts.ReuseDependencyScannerCache) {
1276-
// For the time being, incremental scanner cache validation
1277-
// is not compatible with CAS-enabled builds.
1278-
if (!ctx.CASOpts.EnableCaching) {
1279-
auto cachePath = opts.SerializedDependencyScannerCachePath;
1280-
if (opts.EmitDependencyScannerCacheRemarks)
1281-
ctx.Diags.diagnose(SourceLoc(), diag::remark_reuse_cache, cachePath);
1282-
1283-
llvm::sys::TimePoint<> serializedCacheTimeStamp;
1284-
bool loadFailure = module_dependency_cache_serialization::
1285-
readInterModuleDependenciesCache(cachePath, cache,
1286-
serializedCacheTimeStamp);
1287-
if (opts.EmitDependencyScannerCacheRemarks && loadFailure)
1288-
ctx.Diags.diagnose(SourceLoc(), diag::warn_scanner_deserialize_failed,
1289-
cachePath);
1290-
1291-
if (!loadFailure && opts.ValidatePriorDependencyScannerCache) {
1292-
auto mainModuleID =
1293-
ModuleDependencyID{instance.getMainModule()->getNameStr().str(),
1294-
ModuleDependencyKind::SwiftSource};
1295-
incremental::validateInterModuleDependenciesCache(
1296-
mainModuleID, cache, serializedCacheTimeStamp,
1297-
*instance.getSourceMgr().getFileSystem(), ctx.Diags,
1298-
opts.EmitDependencyScannerCacheRemarks);
1299-
}
1300-
} else if (opts.EmitDependencyScannerCacheRemarks)
1301-
ctx.Diags.diagnose(SourceLoc(), diag::remark_cache_reuse_disabled_with_cas);
1277+
auto cachePath = opts.SerializedDependencyScannerCachePath;
1278+
if (opts.EmitDependencyScannerCacheRemarks)
1279+
ctx.Diags.diagnose(SourceLoc(), diag::remark_reuse_cache, cachePath);
1280+
1281+
llvm::sys::TimePoint<> serializedCacheTimeStamp;
1282+
bool loadFailure =
1283+
module_dependency_cache_serialization::readInterModuleDependenciesCache(
1284+
cachePath, cache, serializedCacheTimeStamp);
1285+
if (opts.EmitDependencyScannerCacheRemarks && loadFailure)
1286+
ctx.Diags.diagnose(SourceLoc(), diag::warn_scanner_deserialize_failed,
1287+
cachePath);
1288+
1289+
if (!loadFailure && opts.ValidatePriorDependencyScannerCache) {
1290+
auto mainModuleID =
1291+
ModuleDependencyID{instance.getMainModule()->getNameStr().str(),
1292+
ModuleDependencyKind::SwiftSource};
1293+
incremental::validateInterModuleDependenciesCache(
1294+
mainModuleID, cache, serializedCacheTimeStamp,
1295+
*instance.getSourceMgr().getFileSystem(), ctx.Diags,
1296+
opts.EmitDependencyScannerCacheRemarks);
1297+
}
13021298
}
13031299

13041300
auto scanner = ModuleDependencyScanner(
@@ -1334,7 +1330,7 @@ swift::dependencies::performModuleScan(
13341330

13351331
// Serialize the dependency cache if -serialize-dependency-scan-cache
13361332
// is specified
1337-
if (opts.SerializeDependencyScannerCache && !ctx.CASOpts.EnableCaching) {
1333+
if (opts.SerializeDependencyScannerCache) {
13381334
auto savePath = opts.SerializedDependencyScannerCachePath;
13391335
module_dependency_cache_serialization::writeInterModuleDependenciesCache(
13401336
ctx.Diags, instance.getOutputBackend(), savePath, cache);
@@ -1449,6 +1445,41 @@ bool swift::dependencies::incremental::verifyModuleDependencyUpToDate(
14491445
return true;
14501446
};
14511447

1448+
auto verifyCASID = [&cache, &diags, emitRemarks](StringRef moduleName,
1449+
const std::string &casID) {
1450+
if (!cache.getScanService().hasCAS()) {
1451+
// If the wrong cache is passed.
1452+
if (emitRemarks)
1453+
diags.diagnose(SourceLoc(),
1454+
diag::remark_scanner_invalidate_configuration,
1455+
moduleName);
1456+
return false;
1457+
}
1458+
auto &CAS = cache.getScanService().getCAS();
1459+
auto ID = CAS.parseID(casID);
1460+
if (!ID) {
1461+
if (emitRemarks)
1462+
diags.diagnose(SourceLoc(), diag::remark_scanner_invalidate_cas_error,
1463+
moduleName, toString(ID.takeError()));
1464+
return false;
1465+
}
1466+
if (!CAS.getReference(*ID)) {
1467+
if (emitRemarks)
1468+
diags.diagnose(SourceLoc(), diag::remark_scanner_invalidate_missing_cas,
1469+
moduleName, casID);
1470+
return false;
1471+
}
1472+
return true;
1473+
};
1474+
1475+
// Check CAS inputs exist
1476+
if (const auto casID = moduleInfo.getClangIncludeTree())
1477+
if (!verifyCASID(moduleID.ModuleName, *casID))
1478+
return false;
1479+
if (const auto casID = moduleInfo.getCASFSRootID())
1480+
if (!verifyCASID(moduleID.ModuleName, *casID))
1481+
return false;
1482+
14521483
// Check interface file for Swift textual modules
14531484
if (const auto &textualModuleDetails = moduleInfo.getAsSwiftInterfaceModule())
14541485
if (!verifyInputOlderThanCacheTimeStamp(

‎test/CAS/incremental_scan.swift

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// RUN: %empty-directory(%t)
2+
// RUN: %empty-directory(%t/module-cache)
3+
4+
// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-cache-path %t/module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/Swift -cache-compile-job -cas-path %t/cas -no-clang-include-tree -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -serialize-dependency-scan-cache -validate-prior-dependency-scan-cache 2>&1 | %FileCheck %s --check-prefix=REUSE --check-prefix=FAILED-LOAD
5+
6+
/// Test reuse
7+
// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-cache-path %t/module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/Swift -cache-compile-job -cas-path %t/cas -no-clang-include-tree -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -serialize-dependency-scan-cache -validate-prior-dependency-scan-cache 2>&1 | %FileCheck %s --check-prefix=REUSE
8+
9+
/// Test invalidation after removing CAS.
10+
// RUN: rm -rf %t/cas
11+
// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-cache-path %t/module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/Swift -cache-compile-job -cas-path %t/cas -no-clang-include-tree -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -serialize-dependency-scan-cache -validate-prior-dependency-scan-cache 2>&1 | %FileCheck %s --check-prefix=REUSE --check-prefix=INVALIDATE
12+
13+
// REUSE: Incremental module scan: Re-using serialized module scanning dependency cache from:
14+
// FAILED-LOD: Incremental module scan: Failed to load module scanning dependency cache from
15+
// INVALIDATE: Incremental module scan: Dependency info for module '{{.*}}' invalidated due to missing CAS input
16+
// INVALIDATE: Incremental module scan: Dependency info for module '{{.*}}' invalidated due to an out-of-date dependency.
17+
// REUSE: Incremental module scan: Serializing module scanning dependency cache to
18+
import E

‎test/CAS/no_incremental_scan_for_you.swift

Lines changed: 0 additions & 8 deletions
This file was deleted.

0 commit comments

Comments
 (0)