[xcodegen] Add buildable folder support

utils/swift-xcodegen/README.md

@@ -87,6 +87,18 @@ PROJECT CONFIGURATION:
                           on the build arguments of surrounding files. This is mainly useful for
                           files that aren't built in the default config, but are still useful to
                           edit (e.g sourcekitdAPI-InProc.cpp). (default: --infer-args)
+  --prefer-folder-refs/--no-prefer-folder-refs
+                          Whether to prefer folder references for groups containing non-source
+                          files (default: --no-prefer-folder-refs)
+  --buildable-folders/--no-buildable-folders
+                          Requires Xcode 16: Enables the use of "buildable folders", allowing
+                          folder references to be used for compatible targets. This allows new
+                          source files to be added to a target without needing to regenerate the
+                          project.
+
+                          Only supported for targets that have no per-file build settings. This
+                          unfortunately means some Clang targes such as 'lib/Basic' and 'stdlib'
+                          cannot currently use buildable folders. (default: --no-buildable-folders)
 
 MISC:
   --project-root-dir <project-root-dir>

utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/ClangBuildArgsProvider.swift

@@ -83,6 +83,11 @@ struct ClangBuildArgsProvider {
     return .init(for: .clang, args: fileArgs.sorted())
   }
 
+  /// Whether the given path has any unique args not covered by `parent`.
+  func hasUniqueArgs(for path: RelativePath, parent: RelativePath) -> Bool {
+    args.hasUniqueArgs(for: path, parent: parent)
+  }
+
   /// Whether the given file has build arguments.
   func hasBuildArgs(for path: RelativePath) -> Bool {
     !args.getArgs(for: path).isEmpty

utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/CommandArgTree.swift

@@ -46,4 +46,14 @@ struct CommandArgTree {
   ) -> Set<Command.Argument> {
     getArgs(for: path).subtracting(getArgs(for: parent))
   }
+
+  /// Whether the given path has any unique args not covered by `parent`.
+  func hasUniqueArgs(for path: RelativePath, parent: RelativePath) -> Bool {
+    let args = getArgs(for: path)
+    guard !args.isEmpty else { return false }
+    // Assuming `parent` is an ancestor of path, the arguments for parent is
+    // guaranteed to be a subset of the arguments for `path`. As such, we
+    // only have to compare sizes here.
+    return args.count != getArgs(for: parent).count
+  }
 }

utils/swift-xcodegen/Sources/SwiftXcodeGen/Generator/ProjectGenerator.swift

@@ -25,7 +25,13 @@ fileprivate final class ProjectGenerator {
   private var project = Xcode.Project()
   private let allTarget: Xcode.Target
 
-  private var groups: [RelativePath: Xcode.Group] = [:]
+  enum CachedGroup {
+    /// Covered by a parent folder reference.
+    case covered
+    /// Present in the project.
+    case present(Xcode.Group)
+  }
+  private var groups: [RelativePath: CachedGroup] = [:]
   private var files: [RelativePath: Xcode.FileReference] = [:]
   private var targets: [String: Xcode.Target] = [:]
   private var unbuildableSources: [ClangTarget.Source] = []
@@ -103,7 +109,7 @@ fileprivate final class ProjectGenerator {
   /// for a file path relative to the project root.
   private func parentGroup(
     for path: RelativePath
-  ) -> (parentGroup: Xcode.Group, childPath: RelativePath) {
+  ) -> (parentGroup: Xcode.Group, childPath: RelativePath)? {
     guard let parent = path.parentDir else {
       // We've already handled paths under the repo, so this must be for
       // paths outside the repo.
@@ -114,18 +120,31 @@ fileprivate final class ProjectGenerator {
     if parent == repoRelativePath || parent == mainRepoDirInProject {
       return (project.mainGroup, path)
     }
-    return (group(for: parent), RelativePath(path.fileName))
+    guard let parentGroup = group(for: parent) else { return nil }
+    return (parentGroup, RelativePath(path.fileName))
   }
 
-  private func group(for path: RelativePath) -> Xcode.Group {
-    if let group = groups[path] {
-      return group
+  /// Returns the group for a given path, or `nil` if the path is covered
+  /// by a parent folder reference.
+  private func group(for path: RelativePath) -> Xcode.Group? {
+    if let result = groups[path] {
+      switch result {
+      case .covered:
+        return nil
+      case .present(let g):
+        return g
+      }
+    }
+    guard
+      files[path] == nil, let (parentGroup, childPath) = parentGroup(for: path)
+    else {
+      groups[path] = .covered
+      return nil
     }
-    let (parentGroup, childPath) = parentGroup(for: path)
     let group = parentGroup.addGroup(
       path: childPath.rawPath, pathBase: .groupDir, name: path.fileName
     )
-    groups[path] = group
+    groups[path] = .present(group)
     return group
   }
 
@@ -163,11 +182,12 @@ fileprivate final class ProjectGenerator {
     // group there.
     if ref.kind == .folder {
       guard groups[path] == nil else {
-        log.warning("Skipping blue folder '\(path)'; already added")
         return nil
       }
     }
-    let (parentGroup, childPath) = parentGroup(for: path)
+    guard let (parentGroup, childPath) = parentGroup(for: path) else {
+      return nil
+    }
     let file = parentGroup.addFileReference(
       path: childPath.rawPath, isDirectory: ref.kind == .folder,
       pathBase: .groupDir, name: path.fileName
@@ -178,10 +198,10 @@ fileprivate final class ProjectGenerator {
 
   @discardableResult
   private func getOrCreateRepoRef(
-    _ ref: ProjectSpec.PathReference, allowExcluded: Bool = false
+    _ ref: ProjectSpec.PathReference
   ) -> Xcode.FileReference? {
     let path = ref.path
-    guard allowExcluded || checkNotExcluded(path) else { return nil }
+    guard checkNotExcluded(path) else { return nil }
     return getOrCreateProjectRef(ref.withPath(repoRelativePath.appending(path)))
   }
 
@@ -190,18 +210,35 @@ fileprivate final class ProjectGenerator {
   }
 
   func generateBaseTarget(
-    _ name: String, productType: Xcode.Target.ProductType?,
-    includeInAllTarget: Bool
+    _ name: String, at parentPath: RelativePath?, canUseBuildableFolder: Bool,
+    productType: Xcode.Target.ProductType?, includeInAllTarget: Bool
   ) -> Xcode.Target? {
     guard targets[name] == nil else {
       log.warning("Duplicate target '\(name)', skipping")
       return nil
     }
+    var buildableFolder: Xcode.FileReference?
+    if let parentPath, !parentPath.components.isEmpty {
+      // If we've been asked to use buildable folders, see if we can create
+      // a folder reference at the parent path. Otherwise, create a group at
+      // the parent path. If we can't create either a folder or group, this is
+      // nested in a folder reference and there's nothing we can do.
+      if spec.useBuildableFolders && canUseBuildableFolder {
+        buildableFolder = getOrCreateRepoRef(.folder(parentPath))
+      }
+      guard buildableFolder != nil ||
+              group(for: repoRelativePath.appending(parentPath)) != nil else {
+        return nil
+      }
+    }
     let target = project.addTarget(productType: productType, name: name)
     targets[name] = target
     if includeInAllTarget {
       allTarget.addDependency(on: target)
     }
+    if let buildableFolder {
+      target.addBuildableFolder(buildableFolder)
+    }
     target.buildSettings.common.ONLY_ACTIVE_ARCH = "YES"
     target.buildSettings.common.USE_HEADERMAP = "NO"
     // The product name needs to be unique across every project we generate
@@ -247,8 +284,12 @@ fileprivate final class ProjectGenerator {
     }
     unbuildableSources += targetInfo.unbuildableSources
 
-    for header in targetInfo.headers {
-      getOrCreateRepoRef(.file(header))
+    // Need to defer the addition of headers since the target may want to use
+    // a buildable folder.
+    defer {
+      for header in targetInfo.headers {
+        getOrCreateRepoRef(.file(header))
+      }
     }
 
     // If we have no sources, we're done.
@@ -262,8 +303,20 @@ fileprivate final class ProjectGenerator {
       }
       return
     }
+    // Can only use buildable folders if there are no unique arguments and no
+    // unbuildable sources.
+    // TODO: To improve the coverage of buildable folders, we ought to start
+    // automatically splitting umbrella Clang targets like 'stdlib', since
+    // they always have files with unique args.
+    let canUseBuildableFolders =
+      try spec.useBuildableFolders && targetInfo.unbuildableSources.isEmpty &&
+      targetInfo.sources.allSatisfy {
+        try !buildDir.clangArgs.hasUniqueArgs(for: $0.path, parent: targetPath)
+      }
+
     let target = generateBaseTarget(
-      targetInfo.name, productType: .staticArchive,
+      targetInfo.name, at: targetPath,
+      canUseBuildableFolder: canUseBuildableFolders, productType: .staticArchive,
       includeInAllTarget: includeInAllTarget
     )
     guard let target else { return }
@@ -437,7 +490,8 @@ fileprivate final class ProjectGenerator {
       )
     }
     let target = generateBaseTarget(
-      targetInfo.name, productType: nil, includeInAllTarget: includeInAllTarget
+      targetInfo.name, at: nil, canUseBuildableFolder: false, productType: nil,
+      includeInAllTarget: includeInAllTarget
     )
     guard let target else { return nil }
 
@@ -477,9 +531,11 @@ fileprivate final class ProjectGenerator {
     guard checkNotExcluded(buildRule.parentPath, for: "Swift target") else {
       return nil
     }
+    // Create the target. Swift targets can always use buildable folders
+    // since they have a consistent set of arguments.
     let target = generateBaseTarget(
-      targetInfo.name, productType: .staticArchive,
-      includeInAllTarget: includeInAllTarget
+      targetInfo.name, at: buildRule.parentPath, canUseBuildableFolder: true,
+      productType: .staticArchive, includeInAllTarget: includeInAllTarget
     )
     guard let target else { return nil }
 
@@ -599,6 +655,11 @@ fileprivate final class ProjectGenerator {
     guard !generated else { return }
     generated = true
 
+    // First add file/folder references.
+    for ref in spec.referencesToAdd {
+      getOrCreateRepoRef(ref)
+    }
+
     // Gather the Swift targets to generate, including any dependencies.
     var swiftTargets: Set<SwiftTarget> = []
     for targetSource in spec.swiftTargetSources {
@@ -681,11 +742,6 @@ fileprivate final class ProjectGenerator {
       }
     }
 
-    for ref in spec.referencesToAdd {
-      // Allow important references to bypass exclusion checks.
-      getOrCreateRepoRef(ref, allowExcluded: ref.isImportant)
-    }
-
     // Sort the groups.
     sortGroupChildren(project.mainGroup)
   }

utils/swift-xcodegen/Sources/SwiftXcodeGen/Generator/ProjectSpec.swift

@@ -36,6 +36,13 @@ public struct ProjectSpec {
   /// on the build arguments of surrounding files.
   public var inferArgs: Bool
 
+  /// Whether to prefer using folder references for groups containing non-source
+  /// files.
+  public var preferFolderRefs: Bool
+
+  /// Whether to enable the use of buildable folders for targets.
+  public var useBuildableFolders: Bool
+
   /// If provided, the paths added will be implicitly appended to this path.
   let mainRepoDir: RelativePath?
 
@@ -50,8 +57,8 @@ public struct ProjectSpec {
     _ name: String, for buildDir: RepoBuildDir, runnableBuildDir: RepoBuildDir,
     addClangTargets: Bool, addSwiftTargets: Bool,
     addSwiftDependencies: Bool, addRunnableTargets: Bool,
-    addBuildForRunnableTargets: Bool, inferArgs: Bool,
-    mainRepoDir: RelativePath? = nil
+    addBuildForRunnableTargets: Bool, inferArgs: Bool, preferFolderRefs: Bool,
+    useBuildableFolders: Bool, mainRepoDir: RelativePath? = nil
   ) {
     self.name = name
     self.buildDir = buildDir
@@ -62,6 +69,8 @@ public struct ProjectSpec {
     self.addRunnableTargets = addRunnableTargets
     self.addBuildForRunnableTargets = addBuildForRunnableTargets
     self.inferArgs = inferArgs
+    self.preferFolderRefs = preferFolderRefs
+    self.useBuildableFolders = useBuildableFolders
     self.mainRepoDir = mainRepoDir
   }
 
@@ -83,14 +92,11 @@ extension ProjectSpec {
     var kind: Kind
     var path: RelativePath
 
-    /// Whether this reference should bypass exclusion checks.
-    var isImportant: Bool
-
-    static func file(_ path: RelativePath, isImportant: Bool = false) -> Self {
-      .init(kind: .file, path: path, isImportant: isImportant)
+    static func file(_ path: RelativePath) -> Self {
+      .init(kind: .file, path: path)
     }
-    static func folder(_ path: RelativePath, isImportant: Bool = false) -> Self {
-      .init(kind: .folder, path: path, isImportant: isImportant)
+    static func folder(_ path: RelativePath) -> Self {
+      .init(kind: .folder, path: path)
     }
 
     func withPath(_ newPath: RelativePath) -> Self {
@@ -145,24 +151,18 @@ extension ProjectSpec {
     self.knownUnbuildables.insert(path)
   }
 
-  public mutating func addReference(
-    to path: RelativePath, isImportant: Bool = false
-  ) {
+  public mutating func addReference(to path: RelativePath) {
     guard let path = mapPath(path, for: "file") else { return }
-    if repoRoot.appending(path).isDirectory {
-      if isImportant {
-        // Important folder references should block anything being added under
-        // them.
-        excludedPaths.append(.init(path: path))
-      }
-      referencesToAdd.append(.folder(path, isImportant: isImportant))
-    } else {
-      referencesToAdd.append(.file(path, isImportant: isImportant))
-    }
+    let isDir = repoRoot.appending(path).isDirectory
+    referencesToAdd.append(isDir ? .folder(path) : .file(path))
   }
 
   public mutating func addHeaders(in path: RelativePath) {
     guard let path = mapPath(path, for: "headers") else { return }
+    if preferFolderRefs {
+      referencesToAdd.append(.folder(path))
+      return
+    }
     do {
       for header in try buildDir.getHeaderFilePaths(for: path) {
         referencesToAdd.append(.file(header))
@@ -184,6 +184,10 @@ extension ProjectSpec {
 
   public mutating func addDocsGroup(at path: RelativePath) {
     guard let path = mapPath(path, for: "docs") else { return }
+    if preferFolderRefs {
+      referencesToAdd.append(.folder(path))
+      return
+    }
     do {
       for doc in try buildDir.getAllRepoSubpaths(of: path) where doc.isDocLike {
         referencesToAdd.append(.file(doc))

utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/PathProtocol.swift

@@ -127,8 +127,11 @@ extension PathProtocol {
 }
 
 extension Collection where Element: PathProtocol {
+  /// Computes the common parent for a collection of paths. If there is only
+  /// a single unique path, this returns the parent for that path.
   var commonAncestor: Element? {
     guard let first = self.first else { return nil }
-    return dropFirst().reduce(first, { $0.commonAncestor(with: $1) })
+    let result = dropFirst().reduce(first, { $0.commonAncestor(with: $1) })
+    return result == first ? result.parentDir : result
   }
 }