SR-1741: Add non-source files from source directories and root, ignore files from .gitignore

Sources/Xcodeproj/generate().swift

@@ -13,6 +13,7 @@ import POSIX
 import PackageGraph
 import PackageModel
 import PackageLoading
+import SourceControl
 import Utility
 
 public struct XcodeprojOptions {
@@ -63,6 +64,7 @@ public func generate(
     projectName: String,
     xcodeprojPath: AbsolutePath,
     graph: PackageGraph,
+    repositoryProvider: RepositoryProvider = GitRepositoryProvider(),
     options: XcodeprojOptions,
     diagnostics: DiagnosticsEngine
 ) throws -> Xcode.Project {
@@ -82,10 +84,16 @@ public func generate(
     // references in the project.
     let extraDirs = try findDirectoryReferences(path: srcroot)
 
+    var extraFiles = [AbsolutePath]()
+    if try repositoryProvider.checkoutExists(at: srcroot) {
+        let workingCheckout = try repositoryProvider.openCheckout(at: srcroot)
+        extraFiles = try getExtraFilesFor(package: graph.rootPackages[0], in: workingCheckout)
+    }
+
     /// Generate the contents of project.xcodeproj (inside the .xcodeproj).
     // FIXME: This could be more efficient by directly writing to a stream
     // instead of first creating a string.
-    let project = try pbxproj(xcodeprojPath: xcodeprojPath, graph: graph, extraDirs: extraDirs, options: options, diagnostics: diagnostics)
+    let project = try pbxproj(xcodeprojPath: xcodeprojPath, graph: graph, extraDirs: extraDirs, extraFiles: extraFiles, options: options, diagnostics: diagnostics)
     try open(xcodeprojPath.appending(component: "project.pbxproj")) { stream in
         // Serialize the project model we created to a plist, and return
         // its string description.
@@ -223,3 +231,40 @@ func generateSchemes(
         ).generate()
     }
 }
+
+// Find and return non-source files in the source directories and root that should be added
+// as a reference to the project.
+func getExtraFilesFor(package: ResolvedPackage, in workingCheckout: WorkingCheckout) throws -> [AbsolutePath] {
+    let srcroot = package.path
+    var extraFiles = try findNonSourceFiles(path: srcroot)
+
+    for target in package.targets {
+        let sourcesDirectory = target.sources.root
+        if localFileSystem.isDirectory(sourcesDirectory) {
+            let sourcesExtraFiles = try findNonSourceFiles(path: sourcesDirectory, recursively: true)
+            extraFiles.append(contentsOf: sourcesExtraFiles)
+        }
+    }
+
+    let isIgnored = try workingCheckout.areIgnored(extraFiles)
+    extraFiles = extraFiles.enumerated().filter({ !isIgnored[$0.offset] }).map({ $0.element })
+
+    return extraFiles
+}
+
+/// Finds the non-source files from `path`
+/// - parameters:
+///   - path: The path of the directory to get the files from
+///   - recursively: Specifies if the directory at `path` should be searched recursively
+func findNonSourceFiles(path: AbsolutePath, recursively: Bool = false) throws -> [AbsolutePath] {
+    let filesFromPath = try walk(path, recursively: recursively)
+
+    return filesFromPath.filter({
+        if !isFile($0) { return false }
+        if $0.basename.hasPrefix(".") { return false }
+        if let `extension` = $0.extension, SupportedLanguageExtension.validExtensions.contains(`extension`) {
+            return false
+        }
+        return true
+    })
+}

Sources/Xcodeproj/pbxproj().swift

@@ -32,6 +32,7 @@ public func pbxproj(
         xcodeprojPath: AbsolutePath,
         graph: PackageGraph,
         extraDirs: [AbsolutePath],
+        extraFiles: [AbsolutePath],
         options: XcodeprojOptions,
         diagnostics: DiagnosticsEngine,
         fileSystem: FileSystem = localFileSystem
@@ -40,6 +41,7 @@ public func pbxproj(
         xcodeprojPath: xcodeprojPath,
         graph: graph,
         extraDirs: extraDirs,
+        extraFiles: extraFiles,
         options: options,
         fileSystem: fileSystem,
         diagnostics: diagnostics)
@@ -55,6 +57,7 @@ func xcodeProject(
     xcodeprojPath: AbsolutePath,
     graph: PackageGraph,
     extraDirs: [AbsolutePath],
+    extraFiles: [AbsolutePath],
     options: XcodeprojOptions,
     fileSystem: FileSystem,
     diagnostics: DiagnosticsEngine,
@@ -322,6 +325,11 @@ func xcodeProject(
         project.mainGroup.addFileReference(path: extraDir.relative(to: sourceRootDir).asString, pathBase: .projectDir)
     }
 
+    for extraFile in extraFiles {
+        let groupOfFile = srcPathsToGroups[extraFile.parentDirectory]
+        groupOfFile?.addFileReference(path: extraFile.basename)
+    }
+
     // Determine the set of targets to generate in the project by excluding
     // any system targets.
     let targets = graph.reachableTargets.filter({ $0.type != .systemModule })

Tests/XcodeprojTests/GenerateXcodeprojTests.swift

@@ -13,6 +13,7 @@ import TestSupport
 import PackageDescription
 import PackageGraph
 import PackageModel
+import SourceControl
 @testable import Xcodeproj
 import Utility
 import XCTest
@@ -76,7 +77,7 @@ class GenerateXcodeprojTests: XCTestCase {
         let options = XcodeprojOptions(xcconfigOverrides: AbsolutePath("/doesntexist"))
         do {
             _ = try xcodeProject(xcodeprojPath: AbsolutePath.root.appending(component: "xcodeproj"),
-                                 graph: graph, extraDirs: [], options: options, fileSystem: fileSystem, diagnostics: diagnostics)
+                                 graph: graph, extraDirs: [], extraFiles: [], options: options, fileSystem: fileSystem, diagnostics: diagnostics)
             XCTFail("Project generation should have failed")
         } catch ProjectGenerationError.xcconfigOverrideNotFound(let path) {
             XCTAssertEqual(options.xcconfigOverrides, path)
@@ -94,16 +95,144 @@ class GenerateXcodeprojTests: XCTestCase {
         XCTAssertFalse(diagnostics.hasErrors)
 
         _ = try xcodeProject(xcodeprojPath: AbsolutePath.root.appending(component: "xcodeproj"),
-                             graph: graph, extraDirs: [], options: XcodeprojOptions(), fileSystem: fileSystem, diagnostics: diagnostics,
+                             graph: graph, extraDirs: [], extraFiles: [], options: XcodeprojOptions(), fileSystem: fileSystem, diagnostics: diagnostics,
                              warningStream: warningStream)
 
         let warnings = warningStream.bytes.asReadableString
         XCTAssertTrue(warnings.contains("warning: Target '\(moduleName)' conflicts with required framework filenames, rename this target to avoid conflicts."))
     }
 
+    func testGenerateXcodeprojWithoutGitRepo() {
+        mktmpdir { dstdir in
+
+            let packagePath = dstdir.appending(component: "Foo")
+            let modulePath = packagePath.appending(components: "Sources", "DummyModule")
+            try makeDirectories(modulePath)
+            try localFileSystem.writeFileContents(modulePath.appending(component: "dummy.swift"), bytes: "dummy_data")
+            try localFileSystem.writeFileContents(packagePath.appending(component: "a.txt"), bytes: "dummy_data")
+
+            let diagnostics = DiagnosticsEngine()
+            let graph = loadMockPackageGraph([packagePath.asString: Package(name: "Foo")], root: packagePath.asString, diagnostics: diagnostics, in: localFileSystem)
+            XCTAssertFalse(diagnostics.hasErrors)
+
+            let projectName = "DummyProjectName"
+            let outpath = Xcodeproj.buildXcodeprojPath(outputDir: dstdir, projectName: projectName)
+            let project = try Xcodeproj.generate(projectName: projectName, xcodeprojPath: outpath, graph: graph, options: XcodeprojOptions(), diagnostics: diagnostics)
+
+            XCTAssertFalse(project.mainGroup.subitems.contains { $0.path == "a.txt" })
+        }
+    }
+
+    func testGenerateXcodeprojWithDotFiles() {
+        mktmpdir { dstdir in
+
+            let packagePath = dstdir.appending(component: "Foo")
+            let modulePath = packagePath.appending(components: "Sources", "DummyModule")
+            try makeDirectories(modulePath)
+            try localFileSystem.writeFileContents(modulePath.appending(component: "dummy.swift"), bytes: "dummy_data")
+            try localFileSystem.writeFileContents(packagePath.appending(component: ".a.txt"), bytes: "dummy_data")
+
+            initGitRepo(packagePath, addFile: false)
+
+            let diagnostics = DiagnosticsEngine()
+            let graph = loadMockPackageGraph([packagePath.asString: Package(name: "Foo")], root: packagePath.asString, diagnostics: diagnostics, in: localFileSystem)
+            XCTAssertFalse(diagnostics.hasErrors)
+
+            let projectName = "DummyProjectName"
+            let outpath = Xcodeproj.buildXcodeprojPath(outputDir: dstdir, projectName: projectName)
+            let project = try Xcodeproj.generate(projectName: projectName, xcodeprojPath: outpath, graph: graph, options: XcodeprojOptions(), diagnostics: diagnostics)
+
+            XCTAssertFalse(project.mainGroup.subitems.contains { $0.path == ".a.txt" })
+        }
+    }
+
+    func testGenerateXcodeprojWithRootFiles() {
+        mktmpdir { dstdir in
+
+            let packagePath = dstdir.appending(component: "Foo")
+            let modulePath = packagePath.appending(components: "Sources", "DummyModule")
+            try makeDirectories(modulePath)
+            try localFileSystem.writeFileContents(modulePath.appending(component: "dummy.swift"), bytes: "dummy_data")
+            try localFileSystem.writeFileContents(packagePath.appending(component: "a.txt"), bytes: "dummy_data")
+
+            initGitRepo(packagePath, addFile: false)
+
+            let diagnostics = DiagnosticsEngine()
+            let graph = loadMockPackageGraph([packagePath.asString: Package(name: "Foo")], root: packagePath.asString, diagnostics: diagnostics, in: localFileSystem)
+            XCTAssertFalse(diagnostics.hasErrors)
+
+            let projectName = "DummyProjectName"
+            let outpath = Xcodeproj.buildXcodeprojPath(outputDir: dstdir, projectName: projectName)
+            let project = try Xcodeproj.generate(projectName: projectName, xcodeprojPath: outpath, graph: graph, options: XcodeprojOptions(), diagnostics: diagnostics)
+
+            XCTAssertTrue(project.mainGroup.subitems.contains { $0.path == "a.txt" })
+        }
+    }
+
+    func testGenerateXcodeprojWithNonSourceFilesInSourceDirectories() {
+        mktmpdir { dstdir in
+
+            let packagePath = dstdir.appending(component: "Foo")
+            let modulePath = packagePath.appending(components: "Sources", "DummyModule")
+            try makeDirectories(modulePath)
+            try localFileSystem.writeFileContents(modulePath.appending(component: "dummy.swift"), bytes: "dummy_data")
+            try localFileSystem.writeFileContents(modulePath.appending(component: "a.txt"), bytes: "dummy_data")
+
+            initGitRepo(packagePath, addFile: false)
+
+            let diagnostics = DiagnosticsEngine()
+            let graph = loadMockPackageGraph([packagePath.asString: Package(name: "Foo")], root: packagePath.asString, diagnostics: diagnostics, in: localFileSystem)
+            XCTAssertFalse(diagnostics.hasErrors)
+
+            let projectName = "DummyProjectName"
+            let outpath = Xcodeproj.buildXcodeprojPath(outputDir: dstdir, projectName: projectName)
+            let project = try Xcodeproj.generate(projectName: projectName, xcodeprojPath: outpath, graph: graph, options: XcodeprojOptions(), diagnostics: diagnostics)
+
+            let sources = project.mainGroup.subitems[1] as? Xcode.Group
+            let dummyModule = sources?.subitems[0] as? Xcode.Group
+            let aTxt = dummyModule?.subitems[0]
+
+            XCTAssertEqual(aTxt?.path, "a.txt")
+        }
+    }
+
+    func testGenerateXcodeprojWithFilesIgnoredByGit() {
+        mktmpdir { dstdir in
+
+            let packagePath = dstdir.appending(component: "Foo")
+            let modulePath = packagePath.appending(components: "Sources", "DummyModule")
+
+            try makeDirectories(modulePath)
+            try localFileSystem.writeFileContents(modulePath.appending(component: "dummy.swift"), bytes: "dummy_data")
+
+            initGitRepo(packagePath, addFile: false)
+            // Add a .gitignore
+            try localFileSystem.writeFileContents(packagePath.appending(component: ".gitignore"), bytes: "ignored_file")
+            try localFileSystem.writeFileContents(modulePath.appending(component: "ignored_file"), bytes: "dummy_data")
+            try localFileSystem.writeFileContents(packagePath.appending(component: "ignored_file"), bytes: "dummy_data")
+
+            let diagnostics = DiagnosticsEngine()
+            let graph = loadMockPackageGraph([packagePath.asString: Package(name: "Foo")], root: packagePath.asString, diagnostics: diagnostics, in: localFileSystem)
+            XCTAssertFalse(diagnostics.hasErrors)
+
+            let projectName = "DummyProjectName"
+            let outpath = Xcodeproj.buildXcodeprojPath(outputDir: dstdir, projectName: projectName)
+            let project = try Xcodeproj.generate(projectName: projectName, xcodeprojPath: outpath, graph: graph, options: XcodeprojOptions(), diagnostics: diagnostics)
+
+            let sources = project.mainGroup.subitems[1] as? Xcode.Group
+            let dummyModule = sources?.subitems[0] as? Xcode.Group
+
+            XCTAssertEqual(dummyModule?.subitems.count, 1)
+            XCTAssertFalse(project.mainGroup.subitems.contains { $0.path == "ignored_file" })
+        }
+    }
+
     static var allTests = [
         ("testXcodebuildCanParseIt", testXcodebuildCanParseIt),
         ("testXcconfigOverrideValidatesPath", testXcconfigOverrideValidatesPath),
         ("testGenerateXcodeprojWithInvalidModuleNames", testGenerateXcodeprojWithInvalidModuleNames),
+        ("testGenerateXcodeprojWithRootFiles", testGenerateXcodeprojWithRootFiles),
+        ("testGenerateXcodeprojWithNonSourceFilesInSourceDirectories", testGenerateXcodeprojWithNonSourceFilesInSourceDirectories),
+        ("testGenerateXcodeprojWithFilesIgnoredByGit", testGenerateXcodeprojWithFilesIgnoredByGit),
     ]
 }

Tests/XcodeprojTests/PackageGraphTests.swift

@@ -41,7 +41,7 @@ class PackageGraphTests: XCTestCase {
 
         let options = XcodeprojOptions(xcconfigOverrides: AbsolutePath("/Overrides.xcconfig"))
 
-        let project = try xcodeProject(xcodeprojPath: AbsolutePath.root.appending(component: "xcodeproj"), graph: g, extraDirs: [], options: options, fileSystem: fs, diagnostics: diagnostics)
+        let project = try xcodeProject(xcodeprojPath: AbsolutePath.root.appending(component: "xcodeproj"), graph: g, extraDirs: [], extraFiles: [], options: options, fileSystem: fs, diagnostics: diagnostics)
 
         XcodeProjectTester(project) { result in
             result.check(projectDir: "Bar")
@@ -167,7 +167,7 @@ class PackageGraphTests: XCTestCase {
                 products: [.library(name: "Bar", type: .dynamic, targets: ["Foo"])],
                 targets: [.target(name: "Foo")]),
         ], root: "/Foo", diagnostics: diagnostics, in: fs)
-        let project = try xcodeProject(xcodeprojPath: AbsolutePath("/Foo/build").appending(component: "xcodeproj"), graph: g, extraDirs: [], options: XcodeprojOptions(), fileSystem: fs, diagnostics: diagnostics)
+        let project = try xcodeProject(xcodeprojPath: AbsolutePath("/Foo/build").appending(component: "xcodeproj"), graph: g, extraDirs: [], extraFiles: [], options: XcodeprojOptions(), fileSystem: fs, diagnostics: diagnostics)
         XcodeProjectTester(project) { result in
             result.check(target: "Foo") { targetResult in
                 targetResult.check(productType: .framework)
@@ -193,7 +193,7 @@ class PackageGraphTests: XCTestCase {
       let g = loadMockPackageGraph([
           "/Bar": Package(name: "Bar", targets: [Target(name: "swift", dependencies: ["Sea", "Sea2"])]),
       ], root: "/Bar", diagnostics: diagnostics, in: fs)
-        let project = try xcodeProject(xcodeprojPath: AbsolutePath("/Bar/build").appending(component: "xcodeproj"), graph: g, extraDirs: [], options: XcodeprojOptions(), fileSystem: fs, diagnostics: diagnostics)
+        let project = try xcodeProject(xcodeprojPath: AbsolutePath("/Bar/build").appending(component: "xcodeproj"), graph: g, extraDirs: [], extraFiles: [], options: XcodeprojOptions(), fileSystem: fs, diagnostics: diagnostics)
 
       XcodeProjectTester(project) { result in
           result.check(target: "swift") { targetResult in
@@ -225,7 +225,7 @@ class PackageGraphTests: XCTestCase {
             "/Pkg": Package(name: "Pkg", targets: [Target(name: "LibraryTests", dependencies: ["Library", "HelperTool"])]),
             ], root: "/Pkg", diagnostics: diagnostics, in: fs)
 
-        let project = try xcodeProject(xcodeprojPath: AbsolutePath.root.appending(component: "xcodeproj"), graph: g, extraDirs: [], options: XcodeprojOptions(), fileSystem: fs, diagnostics: diagnostics)
+        let project = try xcodeProject(xcodeprojPath: AbsolutePath.root.appending(component: "xcodeproj"), graph: g, extraDirs: [], extraFiles: [], options: XcodeprojOptions(), fileSystem: fs, diagnostics: diagnostics)
 
         XcodeProjectTester(project) { result in
             result.check(projectDir: "Pkg")
@@ -331,7 +331,7 @@ class PackageGraphTests: XCTestCase {
                  ),
             ], root: "/Foo", diagnostics: diagnostics, in: fs)
 
-            let project = try xcodeProject(xcodeprojPath: AbsolutePath("/Foo").appending(component: "xcodeproj"), graph: graph, extraDirs: [], options: XcodeprojOptions(), fileSystem: fs, diagnostics: diagnostics)
+            let project = try xcodeProject(xcodeprojPath: AbsolutePath("/Foo").appending(component: "xcodeproj"), graph: graph, extraDirs: [], extraFiles: [], options: XcodeprojOptions(), fileSystem: fs, diagnostics: diagnostics)
             XCTAssertNoDiagnostics(diagnostics)
 
             XcodeProjectTester(project) { result in