Skip to content

Commit 0f01cff

Browse files
cachemeifyoucancachemeifyoucan
authored
Merge pull request #79172 from cachemeifyoucan/eng/caching-scanning-result-cas
[Dependency Scanning] Support loading scanner cache state for caching
2 parents 23a049c + ca749de commit 0f01cff

File tree

7 files changed

+99
-41
lines changed

7 files changed

+99
-41
lines changed

‎include/swift/AST/DiagnosticsCommon.def

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -232,8 +232,14 @@ REMARK(remark_scanner_invalidate_upstream, none,
232232
"Incremental module scan: Dependency info for module '%0' invalidated due to an out-of-date"
233233
" dependency.", (StringRef))
234234

235-
REMARK(remark_cache_reuse_disabled_with_cas, none,
236-
"Incremental module scan: Re-using serialized module scanning dependency cache disabled in Caching build", ())
235+
REMARK(remark_scanner_invalidate_configuration, none,
236+
"Incremental module scan: Dependency info for module '%0' invalidated due to wrong configuration.", (StringRef))
237+
238+
REMARK(remark_scanner_invalidate_cas_error, none,
239+
"Incremental module scan: Dependency info for module '%0' invalidated due to cas error: %1", (StringRef, StringRef))
240+
241+
REMARK(remark_scanner_invalidate_missing_cas, none,
242+
"Incremental module scan: Dependency info for module '%0' invalidated due to missing CAS input '%1'.", (StringRef, StringRef))
237243

238244
//------------------------------------------------------------------------------
239245
// MARK: custom attribute diagnostics

‎include/swift/AST/ModuleDependencies.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1117,6 +1117,8 @@ class SwiftDependencyScanningService {
11171117
return Mapper->mapToString(Path);
11181118
}
11191119

1120+
bool hasCAS() const { return (bool)CAS; }
1121+
11201122
/// Setup caching service.
11211123
bool setupCachingDependencyScanningService(CompilerInstance &Instance);
11221124

‎include/swift/DependencyScan/ScanDependencies.h

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,24 @@
1414
#define SWIFT_DEPENDENCY_SCANDEPENDENCIES_H
1515

1616
#include "swift-c/DependencyScan/DependencyScan.h"
17-
#include "llvm/ADT/StringRef.h"
18-
#include "llvm/ADT/StringSet.h"
19-
#include "llvm/Support/Error.h"
17+
#include "swift/AST/DiagnosticEngine.h"
18+
#include "llvm/ADT/StringMap.h"
19+
#include "llvm/Support/Chrono.h"
20+
#include "llvm/Support/ErrorOr.h"
21+
#include <unordered_set>
2022

2123
namespace llvm {
2224
class StringSaver;
23-
}
25+
namespace vfs {
26+
class FileSystem;
27+
} // namespace vfs
28+
} // namespace llvm
2429

2530
namespace swift {
2631

2732
class CompilerInvocation;
2833
class CompilerInstance;
34+
class DiagnosticEngine;
2935
class ModuleDependenciesCache;
3036
struct ModuleDependencyID;
3137
struct ModuleDependencyIDHash;

‎lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1819,6 +1819,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
18191819
.CASBridgingHeaderIncludeTreeRootID);
18201820
addIdentifier(swiftTextDeps->moduleCacheKey);
18211821
addIdentifier(swiftTextDeps->userModuleVersion);
1822+
addIdentifier(swiftTextDeps->moduleCacheKey);
18221823
break;
18231824
}
18241825
case swift::ModuleDependencyKind::SwiftBinary: {
@@ -1831,6 +1832,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
18311832
addIdentifier(swiftBinDeps->headerImport);
18321833
addIdentifier(swiftBinDeps->definingModuleInterfacePath);
18331834
addIdentifier(swiftBinDeps->userModuleVersion);
1835+
addIdentifier(swiftBinDeps->moduleCacheKey);
18341836
addStringArray(moduleID,
18351837
ModuleIdentifierArrayKind::HeaderInputModuleDependencies,
18361838
clangHeaderDependencyNames);
@@ -1872,6 +1874,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
18721874
swiftSourceDeps->textualModuleDetails.CASFileSystemRootID);
18731875
addIdentifier(swiftSourceDeps->chainedBridgingHeaderPath);
18741876
addIdentifier(swiftSourceDeps->chainedBridgingHeaderContent);
1877+
addIdentifier(swiftSourceDeps->moduleCacheKey);
18751878
break;
18761879
}
18771880
case swift::ModuleDependencyKind::Clang: {

‎lib/DependencyScan/ScanDependencies.cpp

Lines changed: 58 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
//===----------------------------------------------------------------------===//
1212

1313
#include "swift-c/DependencyScan/DependencyScan.h"
14+
#include "swift/AST/DiagnosticsCommon.h"
1415
#include "swift/Basic/PrettyStackTrace.h"
1516

1617
#include "swift/AST/ASTContext.h"
@@ -1274,32 +1275,27 @@ swift::dependencies::performModuleScan(
12741275
// Load the dependency cache if -reuse-dependency-scan-cache
12751276
// is specified
12761277
if (opts.ReuseDependencyScannerCache) {
1277-
// For the time being, incremental scanner cache validation
1278-
// is not compatible with CAS-enabled builds.
1279-
if (!ctx.CASOpts.EnableCaching) {
1280-
auto cachePath = opts.SerializedDependencyScannerCachePath;
1281-
if (opts.EmitDependencyScannerCacheRemarks)
1282-
ctx.Diags.diagnose(SourceLoc(), diag::remark_reuse_cache, cachePath);
1283-
1284-
llvm::sys::TimePoint<> serializedCacheTimeStamp;
1285-
bool loadFailure = module_dependency_cache_serialization::
1286-
readInterModuleDependenciesCache(cachePath, cache,
1287-
serializedCacheTimeStamp);
1288-
if (opts.EmitDependencyScannerCacheRemarks && loadFailure)
1289-
ctx.Diags.diagnose(SourceLoc(), diag::warn_scanner_deserialize_failed,
1290-
cachePath);
1291-
1292-
if (!loadFailure && opts.ValidatePriorDependencyScannerCache) {
1293-
auto mainModuleID =
1294-
ModuleDependencyID{instance.getMainModule()->getNameStr().str(),
1295-
ModuleDependencyKind::SwiftSource};
1296-
incremental::validateInterModuleDependenciesCache(
1297-
mainModuleID, cache, serializedCacheTimeStamp,
1298-
*instance.getSourceMgr().getFileSystem(), ctx.Diags,
1299-
opts.EmitDependencyScannerCacheRemarks);
1300-
}
1301-
} else if (opts.EmitDependencyScannerCacheRemarks)
1302-
ctx.Diags.diagnose(SourceLoc(), diag::remark_cache_reuse_disabled_with_cas);
1278+
auto cachePath = opts.SerializedDependencyScannerCachePath;
1279+
if (opts.EmitDependencyScannerCacheRemarks)
1280+
ctx.Diags.diagnose(SourceLoc(), diag::remark_reuse_cache, cachePath);
1281+
1282+
llvm::sys::TimePoint<> serializedCacheTimeStamp;
1283+
bool loadFailure =
1284+
module_dependency_cache_serialization::readInterModuleDependenciesCache(
1285+
cachePath, cache, serializedCacheTimeStamp);
1286+
if (opts.EmitDependencyScannerCacheRemarks && loadFailure)
1287+
ctx.Diags.diagnose(SourceLoc(), diag::warn_scanner_deserialize_failed,
1288+
cachePath);
1289+
1290+
if (!loadFailure && opts.ValidatePriorDependencyScannerCache) {
1291+
auto mainModuleID =
1292+
ModuleDependencyID{instance.getMainModule()->getNameStr().str(),
1293+
ModuleDependencyKind::SwiftSource};
1294+
incremental::validateInterModuleDependenciesCache(
1295+
mainModuleID, cache, serializedCacheTimeStamp,
1296+
*instance.getSourceMgr().getFileSystem(), ctx.Diags,
1297+
opts.EmitDependencyScannerCacheRemarks);
1298+
}
13031299
}
13041300

13051301
auto scanner = ModuleDependencyScanner(
@@ -1335,7 +1331,7 @@ swift::dependencies::performModuleScan(
13351331

13361332
// Serialize the dependency cache if -serialize-dependency-scan-cache
13371333
// is specified
1338-
if (opts.SerializeDependencyScannerCache && !ctx.CASOpts.EnableCaching) {
1334+
if (opts.SerializeDependencyScannerCache) {
13391335
auto savePath = opts.SerializedDependencyScannerCachePath;
13401336
module_dependency_cache_serialization::writeInterModuleDependenciesCache(
13411337
ctx.Diags, instance.getOutputBackend(), savePath, cache);
@@ -1450,6 +1446,41 @@ bool swift::dependencies::incremental::verifyModuleDependencyUpToDate(
14501446
return true;
14511447
};
14521448

1449+
auto verifyCASID = [&cache, &diags, emitRemarks](StringRef moduleName,
1450+
const std::string &casID) {
1451+
if (!cache.getScanService().hasCAS()) {
1452+
// If the wrong cache is passed.
1453+
if (emitRemarks)
1454+
diags.diagnose(SourceLoc(),
1455+
diag::remark_scanner_invalidate_configuration,
1456+
moduleName);
1457+
return false;
1458+
}
1459+
auto &CAS = cache.getScanService().getCAS();
1460+
auto ID = CAS.parseID(casID);
1461+
if (!ID) {
1462+
if (emitRemarks)
1463+
diags.diagnose(SourceLoc(), diag::remark_scanner_invalidate_cas_error,
1464+
moduleName, toString(ID.takeError()));
1465+
return false;
1466+
}
1467+
if (!CAS.getReference(*ID)) {
1468+
if (emitRemarks)
1469+
diags.diagnose(SourceLoc(), diag::remark_scanner_invalidate_missing_cas,
1470+
moduleName, casID);
1471+
return false;
1472+
}
1473+
return true;
1474+
};
1475+
1476+
// Check CAS inputs exist
1477+
if (const auto casID = moduleInfo.getClangIncludeTree())
1478+
if (!verifyCASID(moduleID.ModuleName, *casID))
1479+
return false;
1480+
if (const auto casID = moduleInfo.getCASFSRootID())
1481+
if (!verifyCASID(moduleID.ModuleName, *casID))
1482+
return false;
1483+
14531484
// Check interface file for Swift textual modules
14541485
if (const auto &textualModuleDetails = moduleInfo.getAsSwiftInterfaceModule())
14551486
if (!verifyInputOlderThanCacheTimeStamp(

‎test/CAS/incremental_scan.swift

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// RUN: %empty-directory(%t)
2+
// RUN: %empty-directory(%t/module-cache)
3+
4+
// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-cache-path %t/module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/Swift -cache-compile-job -cas-path %t/cas -no-clang-include-tree -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -serialize-dependency-scan-cache -validate-prior-dependency-scan-cache 2>&1 | %FileCheck %s --check-prefix=REUSE --check-prefix=FAILED-LOAD
5+
6+
/// Test reuse
7+
// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-cache-path %t/module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/Swift -cache-compile-job -cas-path %t/cas -no-clang-include-tree -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -serialize-dependency-scan-cache -validate-prior-dependency-scan-cache 2>&1 | %FileCheck %s --check-prefix=REUSE
8+
9+
/// Test invalidation after removing CAS.
10+
// RUN: rm -rf %t/cas
11+
// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-cache-path %t/module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/Swift -cache-compile-job -cas-path %t/cas -no-clang-include-tree -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -serialize-dependency-scan-cache -validate-prior-dependency-scan-cache 2>&1 | %FileCheck %s --check-prefix=REUSE --check-prefix=INVALIDATE
12+
13+
// REUSE: Incremental module scan: Re-using serialized module scanning dependency cache from:
14+
// FAILED-LOD: Incremental module scan: Failed to load module scanning dependency cache from
15+
// INVALIDATE: Incremental module scan: Dependency info for module '{{.*}}' invalidated due to missing CAS input
16+
// INVALIDATE: Incremental module scan: Dependency info for module '{{.*}}' invalidated due to an out-of-date dependency.
17+
// REUSE: Incremental module scan: Serializing module scanning dependency cache to
18+
import E

‎test/CAS/no_incremental_scan_for_you.swift

Lines changed: 0 additions & 8 deletions
This file was deleted.

0 commit comments

Comments
 (0)