Skip to content

Commit 8e3a620

Browse files
artemcmartemcm
authored
Merge pull request #76915 from artemcm/AsyncScanExperiment
[Dependency Scanning] Parallelize Clang module queries
2 parents 58d2259 + 0f50693 commit 8e3a620

16 files changed

+1220
-741
lines changed

‎include/swift/AST/ModuleDependencies.h

Lines changed: 142 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ class ModuleDependencyInfoStorageBase {
180180
ArrayRef<LinkLibrary> linkLibraries,
181181
StringRef moduleCacheKey = "")
182182
: dependencyKind(dependencyKind), linkLibraries(linkLibraries),
183-
moduleCacheKey(moduleCacheKey.str()), resolved(false),
183+
moduleCacheKey(moduleCacheKey.str()),
184184
finalized(false) {}
185185

186186
ModuleDependencyInfoStorageBase(
@@ -191,7 +191,7 @@ class ModuleDependencyInfoStorageBase {
191191
: dependencyKind(dependencyKind), moduleImports(moduleImports),
192192
optionalModuleImports(optionalModuleImports),
193193
linkLibraries(linkLibraries), moduleCacheKey(moduleCacheKey.str()),
194-
resolved(false), finalized(false) {}
194+
finalized(false) {}
195195

196196
virtual ModuleDependencyInfoStorageBase *clone() const = 0;
197197

@@ -209,11 +209,13 @@ class ModuleDependencyInfoStorageBase {
209209
/// use this module.
210210
std::vector<LinkLibrary> linkLibraries;
211211

212-
/// The set of modules on which this module depends, resolved
213-
/// to Module IDs, qualified by module kind: Swift, Clang, etc.
214-
std::vector<ModuleDependencyID> resolvedDirectModuleDependencies;
215-
216-
/// Dependencies comprised of Swift overlay modules of direct and
212+
/// All directly-imported Swift module dependencies.
213+
std::vector<ModuleDependencyID> importedSwiftModules;
214+
/// All directly-imported Clang module dependencies.
215+
std::vector<ModuleDependencyID> importedClangModules;
216+
/// All cross-import overlay module dependencies.
217+
std::vector<ModuleDependencyID> crossImportOverlayModules;
218+
/// All dependencies comprised of Swift overlay modules of direct and
217219
/// transitive Clang dependencies.
218220
std::vector<ModuleDependencyID> swiftOverlayDependencies;
219221

@@ -227,8 +229,6 @@ class ModuleDependencyInfoStorageBase {
227229
/// The macro dependencies.
228230
std::map<std::string, MacroPluginDependency> macroDependencies;
229231

230-
/// The direct dependency of the module is resolved by scanner.
231-
bool resolved;
232232
/// ModuleDependencyInfo is finalized (with all transitive dependencies
233233
/// and inputs).
234234
bool finalized;
@@ -239,6 +239,8 @@ struct CommonSwiftTextualModuleDependencyDetails {
239239
ArrayRef<StringRef> extraPCMArgs, ArrayRef<StringRef> buildCommandLine,
240240
StringRef CASFileSystemRootID)
241241
: extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()),
242+
bridgingHeaderFile(std::nullopt),
243+
bridgingSourceFiles(), bridgingModuleDependencies(),
242244
buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()),
243245
CASFileSystemRootID(CASFileSystemRootID) {}
244246

@@ -254,7 +256,7 @@ struct CommonSwiftTextualModuleDependencyDetails {
254256
std::vector<std::string> bridgingSourceFiles;
255257

256258
/// (Clang) modules on which the bridging header depends.
257-
std::vector<std::string> bridgingModuleDependencies;
259+
std::vector<ModuleDependencyID> bridgingModuleDependencies;
258260

259261
/// The Swift frontend invocation arguments to build the Swift module from the
260262
/// interface.
@@ -426,7 +428,7 @@ class SwiftBinaryModuleDependencyStorage
426428
std::vector<std::string> headerSourceFiles;
427429

428430
/// (Clang) modules on which the header inputs depend.
429-
std::vector<std::string> headerModuleDependencies;
431+
std::vector<ModuleDependencyID> headerModuleDependencies;
430432

431433
/// A flag that indicates this dependency is a framework
432434
const bool isFramework;
@@ -660,42 +662,105 @@ class ModuleDependencyInfo {
660662
return storage->optionalModuleImports;
661663
}
662664

663-
/// Retreive the module-level dependencies.
664-
const ArrayRef<ModuleDependencyID> getDirectModuleDependencies() const {
665-
assert(storage->resolved);
666-
return storage->resolvedDirectModuleDependencies;
667-
}
668-
669665
std::string getModuleCacheKey() const {
670-
assert(storage->resolved);
671666
return storage->moduleCacheKey;
672667
}
673668

674669
void updateModuleCacheKey(const std::string &key) {
675670
storage->moduleCacheKey = key;
676671
}
677672

678-
/// Resolve a dependency's set of `imports` with qualified Module IDs
679673
void
680-
resolveDirectDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
681-
assert(!storage->resolved && "Resolving an already-resolved dependency");
682-
storage->resolved = true;
683-
storage->resolvedDirectModuleDependencies.assign(dependencyIDs.begin(),
684-
dependencyIDs.end());
674+
setImportedSwiftDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
675+
assert(isSwiftModule());
676+
storage->importedSwiftModules.assign(dependencyIDs.begin(),
677+
dependencyIDs.end());
678+
}
679+
const ArrayRef<ModuleDependencyID> getImportedSwiftDependencies() const {
680+
return storage->importedSwiftModules;
681+
}
682+
683+
void
684+
setImportedClangDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
685+
storage->importedClangModules.assign(dependencyIDs.begin(),
686+
dependencyIDs.end());
687+
}
688+
const ArrayRef<ModuleDependencyID> getImportedClangDependencies() const {
689+
return storage->importedClangModules;
690+
}
691+
692+
void
693+
setHeaderClangDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
694+
assert(isSwiftModule());
695+
switch (getKind()) {
696+
case swift::ModuleDependencyKind::SwiftInterface: {
697+
auto swiftInterfaceStorage =
698+
cast<SwiftInterfaceModuleDependenciesStorage>(storage.get());
699+
swiftInterfaceStorage->textualModuleDetails.bridgingModuleDependencies.assign(dependencyIDs.begin(),
700+
dependencyIDs.end());
701+
break;
702+
}
703+
case swift::ModuleDependencyKind::SwiftSource: {
704+
auto swiftSourceStorage =
705+
cast<SwiftSourceModuleDependenciesStorage>(storage.get());
706+
swiftSourceStorage->textualModuleDetails.bridgingModuleDependencies.assign(dependencyIDs.begin(),
707+
dependencyIDs.end());
708+
break;
709+
}
710+
case swift::ModuleDependencyKind::SwiftBinary: {
711+
auto swiftBinaryStorage =
712+
cast<SwiftBinaryModuleDependencyStorage>(storage.get());
713+
swiftBinaryStorage->headerModuleDependencies.assign(dependencyIDs.begin(),
714+
dependencyIDs.end());
715+
break;
716+
}
717+
default: {
718+
llvm_unreachable("Unexpected dependency kind");
719+
}
720+
}
721+
}
722+
const ArrayRef<ModuleDependencyID> getHeaderClangDependencies() const {
723+
switch (getKind()) {
724+
case swift::ModuleDependencyKind::SwiftInterface: {
725+
auto swiftInterfaceStorage =
726+
cast<SwiftInterfaceModuleDependenciesStorage>(storage.get());
727+
return swiftInterfaceStorage->textualModuleDetails.bridgingModuleDependencies;
728+
}
729+
case swift::ModuleDependencyKind::SwiftSource: {
730+
auto swiftSourceStorage =
731+
cast<SwiftSourceModuleDependenciesStorage>(storage.get());
732+
return swiftSourceStorage->textualModuleDetails.bridgingModuleDependencies;
733+
}
734+
case swift::ModuleDependencyKind::SwiftBinary: {
735+
auto swiftBinaryStorage =
736+
cast<SwiftBinaryModuleDependencyStorage>(storage.get());
737+
return swiftBinaryStorage->headerModuleDependencies;
738+
}
739+
default:
740+
return {};
741+
}
685742
}
686743

687-
/// Set this module's set of Swift Overlay dependencies
688744
void
689-
setOverlayDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
745+
setSwiftOverlayDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
690746
assert(isSwiftModule());
691747
storage->swiftOverlayDependencies.assign(dependencyIDs.begin(),
692748
dependencyIDs.end());
693749
}
694-
695750
const ArrayRef<ModuleDependencyID> getSwiftOverlayDependencies() const {
696751
return storage->swiftOverlayDependencies;
697752
}
698753

754+
void
755+
setCrossImportOverlayDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
756+
assert(isSwiftModule());
757+
storage->crossImportOverlayModules.assign(dependencyIDs.begin(),
758+
dependencyIDs.end());
759+
}
760+
const ArrayRef<ModuleDependencyID> getCrossImportOverlayDependencies() const {
761+
return storage->crossImportOverlayModules;
762+
}
763+
699764
const ArrayRef<LinkLibrary> getLinkLibraries() const {
700765
return storage->linkLibraries;
701766
}
@@ -723,16 +788,6 @@ class ModuleDependencyInfo {
723788
return {};
724789
}
725790

726-
const ArrayRef<std::string> getHeaderDependencies() const {
727-
if (auto *detail = getAsSwiftInterfaceModule())
728-
return detail->textualModuleDetails.bridgingModuleDependencies;
729-
if (auto *detail = getAsSwiftSourceModule())
730-
return detail->textualModuleDetails.bridgingModuleDependencies;
731-
if (auto *detail = getAsSwiftBinaryModule())
732-
return detail->headerModuleDependencies;
733-
return {};
734-
}
735-
736791
std::vector<std::string> getCommandline() const {
737792
if (auto *detail = getAsClangModule())
738793
return detail->buildCommandLine;
@@ -798,9 +853,8 @@ class ModuleDependencyInfo {
798853
llvm_unreachable("Unexpected type");
799854
}
800855

801-
bool isResolved() const { return storage->resolved; }
802-
void setIsResolved(bool isResolved) { storage->resolved = isResolved; }
803-
856+
/// Whether explicit input paths of all the module dependencies
857+
/// have been specified on the command-line recipe for this module.
804858
bool isFinalized() const { return storage->finalized; }
805859
void setIsFinalized(bool isFinalized) { storage->finalized = isFinalized; }
806860

@@ -876,10 +930,6 @@ class ModuleDependencyInfo {
876930
const SourceManager *sourceManager = nullptr,
877931
SourceLoc sourceLocation = SourceLoc());
878932

879-
/// Add a kind-qualified module dependency ID to the set of
880-
/// module dependencies.
881-
void addModuleDependency(ModuleDependencyID dependencyID);
882-
883933
/// Get the bridging header.
884934
std::optional<std::string> getBridgingHeader() const;
885935

@@ -904,10 +954,6 @@ class ModuleDependencyInfo {
904954
/// Add source files that the header input depends on.
905955
void addHeaderSourceFile(StringRef bridgingSourceFile);
906956

907-
/// Add (Clang) modules on which a non-bridging header input depends.
908-
void addHeaderInputModuleDependency(StringRef module,
909-
llvm::StringSet<> &alreadyAddedModules);
910-
911957
/// Add bridging header include tree.
912958
void addBridgingHeaderIncludeTree(StringRef ID);
913959

@@ -1184,6 +1230,8 @@ class ModuleDependenciesCache {
11841230
std::optional<ModuleDependencyKind> kind) const;
11851231
/// Whether we have cached dependency information for the given module Name.
11861232
bool hasDependency(StringRef moduleName) const;
1233+
/// Whether we have cached dependency information for the given Swift module.
1234+
bool hasSwiftDependency(StringRef moduleName) const;
11871235

11881236
SwiftDependencyScanningService &getScanService() {
11891237
return globalScanningService;
@@ -1200,17 +1248,29 @@ class ModuleDependenciesCache {
12001248
}
12011249
std::string getModuleOutputPath() const { return moduleOutputPath; }
12021250

1203-
/// Query all dependencies, direct and Swift overlay.
1204-
std::vector<ModuleDependencyID>
1251+
/// Query all dependencies
1252+
ModuleDependencyIDSetVector
12051253
getAllDependencies(const ModuleDependencyID &moduleID) const;
1206-
1207-
/// Query only direct import dependencies
1254+
1255+
/// Query all Clang module dependencies.
1256+
ModuleDependencyIDSetVector
1257+
getClangDependencies(const ModuleDependencyID &moduleID) const;
1258+
1259+
/// Query all directly-imported Swift dependencies
12081260
llvm::ArrayRef<ModuleDependencyID>
1209-
getOnlyDirectDependencies(const ModuleDependencyID &moduleID) const;
1210-
1211-
/// Query only Swift overlay dependencies
1261+
getImportedSwiftDependencies(const ModuleDependencyID &moduleID) const;
1262+
/// Query all directly-imported Clang dependencies
1263+
llvm::ArrayRef<ModuleDependencyID>
1264+
getImportedClangDependencies(const ModuleDependencyID &moduleID) const;
1265+
/// Query all Clang module dependencies of this module's imported (bridging) header
12121266
llvm::ArrayRef<ModuleDependencyID>
1213-
getOnlyOverlayDependencies(const ModuleDependencyID &moduleID) const;
1267+
getHeaderClangDependencies(const ModuleDependencyID &moduleID) const;
1268+
/// Query Swift overlay dependencies
1269+
llvm::ArrayRef<ModuleDependencyID>
1270+
getSwiftOverlayDependencies(const ModuleDependencyID &moduleID) const;
1271+
/// Query all cross-import overlay dependencies
1272+
llvm::ArrayRef<ModuleDependencyID>
1273+
getCrossImportOverlayDependencies(const ModuleDependencyID &moduleID) const;
12141274

12151275
/// Look for module dependencies for a module with the given ID
12161276
///
@@ -1231,6 +1291,12 @@ class ModuleDependenciesCache {
12311291
std::optional<const ModuleDependencyInfo *>
12321292
findDependency(StringRef moduleName) const;
12331293

1294+
/// Look for Swift module dependencies for a module with the given name
1295+
///
1296+
/// \returns the cached result, or \c None if there is no cached entry.
1297+
std::optional<const ModuleDependencyInfo *>
1298+
findSwiftDependency(StringRef moduleName) const;
1299+
12341300
/// Look for known existing dependencies.
12351301
///
12361302
/// \returns the cached result.
@@ -1246,19 +1312,32 @@ class ModuleDependenciesCache {
12461312

12471313
/// Update stored dependencies for the given module.
12481314
void updateDependency(ModuleDependencyID moduleID,
1249-
ModuleDependencyInfo dependencies);
1315+
ModuleDependencyInfo dependencyInfo);
12501316

1251-
/// Resolve a dependency module's set of imports
1252-
/// to a kind-qualified set of module IDs.
1317+
/// Resolve this module's set of directly-imported Swift module
1318+
/// dependencies
12531319
void
1254-
resolveDependencyImports(ModuleDependencyID moduleID,
1255-
const ArrayRef<ModuleDependencyID> dependencyIDs);
1256-
1257-
/// Resolve a dependency module's set of Swift module dependencies
1320+
setImportedSwiftDependencies(ModuleDependencyID moduleID,
1321+
const ArrayRef<ModuleDependencyID> dependencyIDs);
1322+
/// Resolve this module's set of directly-imported Clang module
1323+
/// dependencies
1324+
void
1325+
setImportedClangDependencies(ModuleDependencyID moduleID,
1326+
const ArrayRef<ModuleDependencyID> dependencyIDs);
1327+
/// Resolve this module's set of Swift module dependencies
12581328
/// that are Swift overlays of Clang module dependencies.
12591329
void
12601330
setSwiftOverlayDependencies(ModuleDependencyID moduleID,
12611331
const ArrayRef<ModuleDependencyID> dependencyIDs);
1332+
/// Resolve this Swift module's imported (bridging) header's
1333+
/// Clang module dependencies
1334+
void
1335+
setHeaderClangDependencies(ModuleDependencyID moduleID,
1336+
const ArrayRef<ModuleDependencyID> dependencyIDs);
1337+
/// Resolve this module's cross-import overlay dependencies
1338+
void
1339+
setCrossImportOverlayDependencies(ModuleDependencyID moduleID,
1340+
const ArrayRef<ModuleDependencyID> dependencyIDs);
12621341

12631342
StringRef getMainModuleName() const { return mainScanModuleName; }
12641343
};

‎include/swift/ClangImporter/ClangImporter.h

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ class SwiftLookupTable;
9090
class TypeDecl;
9191
class ValueDecl;
9292
class VisibleDeclConsumer;
93+
using ModuleDependencyIDSetVector =
94+
llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
95+
std::set<ModuleDependencyID>>;
9396
enum class SelectorSplitKind;
9497

9598
/// Kinds of optional types.
@@ -492,7 +495,11 @@ class ClangImporter final : public ClangModuleLoader {
492495
ModuleDependencyInfo &MDI,
493496
const clang::tooling::dependencies::TranslationUnitDeps &deps);
494497

495-
/// Add dependency information for header dependencies
498+
void getBridgingHeaderOptions(
499+
const clang::tooling::dependencies::TranslationUnitDeps &deps,
500+
std::vector<std::string> &swiftArgs);
501+
502+
/// Query dependency information for header dependencies
496503
/// of a binary Swift module.
497504
///
498505
/// \param moduleID the name of the Swift module whose dependency
@@ -505,10 +512,14 @@ class ClangImporter final : public ClangModuleLoader {
505512
/// about new Clang modules discovered along the way.
506513
///
507514
/// \returns \c true if an error occurred, \c false otherwise
508-
bool addHeaderDependencies(
515+
bool getHeaderDependencies(
509516
ModuleDependencyID moduleID,
510517
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
511-
ModuleDependenciesCache &cache);
518+
ModuleDependenciesCache &cache,
519+
ModuleDependencyIDSetVector &headerClangModuleDependencies,
520+
std::vector<std::string> &headerFileInputs,
521+
std::vector<std::string> &bridgingHeaderCommandLine,
522+
std::optional<std::string> &includeTreeID);
512523

513524
clang::TargetInfo &getModuleAvailabilityTarget() const override;
514525
clang::ASTContext &getClangASTContext() const override;

0 commit comments

Comments
 (0)