[APIDiff] Improve API digester integration & polish of experimental-api-diff (#3485)

* [APIDiff] Parse serialized diagnostics from API digester to differentiate breaking changes from other diags

* [APIDiff] Refactoring and documentation to make the diffing process easier to understand

* [APIDiff] Diff and print results from one module at a time

* [APIDiff] Improve test coverage

* [APIDiff] Use per-module baselines

* [APIDiff] Only diagnose API breakage in modules vended by library products

* [APIDiff] Add a test case involving a C-family target

* [APIDiff] Print a message if a module has no breaking changes

* [APIDiff] Skip comparison of modules which aren't present in the baseline

* [APIDiff] Refactor API digester tool to accept a BuildPlan instead of opaque args

* [APIDiff] Remove an option-parsing hack that's no longer needed

* [APIDiff] Improve failure diagnostics

* [APIDiff] Remove an obsolete test

* [APIDiff] Adapt to a TSC API change

* [APIDiff] Rename SwiftAPIDigester.ComparisonResult.isSuccessful -> SwiftAPIDigester.ComparisonResult.hasNoAPIBreakingChanges

* [APIDiff] Allow restricting the diff to specific products and/or targets

* [APIDiff] Improve subcommand help

* [APIDiff] Parallelize diff operation

* [APIDiff] Parallelize baseline generation

* [APIDiff] Clarify filtering options

* [APIDiff] Control paralellism of api-digester invocations with -j

* Update help text

Author: owenv

Fixtures/Miscellaneous/APIDiff/Bar/Package.swift

@@ -0,0 +1,14 @@
+// swift-tools-version:4.2
+import PackageDescription
+
+let package = Package(
+    name: "Bar",
+    products: [
+        .library(name: "Baz", targets: ["Baz"]),
+        .library(name: "Qux", targets: ["Qux"]),
+    ],
+    targets: [
+        .target(name: "Baz"),
+        .target(name: "Qux")
+    ]
+)

Fixtures/Miscellaneous/APIDiff/Bar/Sources/Baz/Baz.swift

@@ -0,0 +1,3 @@
+public func bar() -> Int {
+    42
+}

Fixtures/Miscellaneous/APIDiff/Bar/Sources/Qux/Qux.swift

@@ -0,0 +1 @@
+public class Qux<T> { public let x = 1 }

Fixtures/Miscellaneous/APIDiff/CTargetDep/Package.swift

@@ -0,0 +1,13 @@
+// swift-tools-version:4.2
+import PackageDescription
+
+let package = Package(
+    name: "CLibrarySources",
+    products: [
+        .library(name: "Lib", targets: ["Bar"])
+    ],
+    targets: [
+        .target(name: "Foo"),
+        .target(name: "Bar", dependencies: ["Foo"])
+    ]
+)

Fixtures/Miscellaneous/APIDiff/CTargetDep/Sources/Bar/Bar.swift

@@ -0,0 +1,6 @@
+import Foo
+
+public func bar() -> Int {
+    foo()
+    return 42
+}

Fixtures/Miscellaneous/APIDiff/CTargetDep/Sources/Foo/Foo.c

@@ -0,0 +1,8 @@
+#include "Foo.h"
+
+int foo() {
+    int a = 5;
+    int b = a;
+    a = b;
+    return a;
+}

Fixtures/Miscellaneous/APIDiff/CTargetDep/Sources/Foo/include/Foo.h

@@ -0,0 +1 @@
+int foo();

Fixtures/Miscellaneous/APIDiff/Foo/Foo.swift

@@ -0,0 +1,3 @@
+public func foo() {
+	{}()
+}

Fixtures/Miscellaneous/APIDiff/Foo/Package.swift

@@ -0,0 +1,12 @@
+// swift-tools-version:4.2
+import PackageDescription
+
+let package = Package(
+    name: "Foo",
+    products: [
+        .library(name: "Foo", targets: ["Foo"]),
+    ],
+    targets: [
+        .target(name: "Foo", path: "./"),
+    ]
+)

Fixtures/Miscellaneous/APIDiff/NonAPILibraryTargets/Package.swift

@@ -0,0 +1,18 @@
+// swift-tools-version:4.2
+import PackageDescription
+
+let package = Package(
+    name: "NonAPILibraryTargets",
+    products: [
+        .library(name: "One", targets: ["Foo"]),
+        .library(name: "Two", targets: ["Bar", "Baz"]),
+        .executable(name: "Exec", targets: ["Exec", "Qux"])
+    ],
+    targets: [
+        .target(name: "Foo"),
+        .target(name: "Bar", dependencies: ["Baz"]),
+        .target(name: "Baz"),
+        .target(name: "Qux"),
+        .target(name: "Exec", dependencies: ["Qux"])
+    ]
+)

Fixtures/Miscellaneous/APIDiff/NonAPILibraryTargets/Sources/Bar/Bar.swift

@@ -0,0 +1,5 @@
+import Baz
+
+public func bar() -> Int {
+    42
+}

Fixtures/Miscellaneous/APIDiff/NonAPILibraryTargets/Sources/Baz/Baz.swift

@@ -0,0 +1,3 @@
+public enum Baz {
+    case a, c
+}

Fixtures/Miscellaneous/APIDiff/NonAPILibraryTargets/Sources/Exec/main.swift

@@ -0,0 +1,3 @@
+import Qux
+
+print("Hello, world!")

Fixtures/Miscellaneous/APIDiff/NonAPILibraryTargets/Sources/Foo/Foo.swift

@@ -0,0 +1,3 @@
+public struct Foo {
+    func doThing() {}
+}

Fixtures/Miscellaneous/APIDiff/NonAPILibraryTargets/Sources/Qux/Qux.swift

@@ -0,0 +1 @@
+public class Qux<T> { public let x = 1 }

Sources/Build/BuildPlan.swift

@@ -1919,14 +1919,7 @@ public class BuildPlan {
         // Add search paths from the system library targets.
         for target in graph.reachableTargets {
             if let systemLib = target.underlyingTarget as? SystemLibraryTarget {
-                for flag in self.pkgConfig(for: systemLib).cFlags {
-                    // The api-digester tool doesn't like `-I<Foo>` style for some reason.
-                    if flag.hasPrefix("-I") && flag.count > 2 {
-                        arguments += ["-I", String(flag.dropFirst(2))]
-                    } else {
-                        arguments.append(flag)
-                    }
-                }
+                arguments.append(contentsOf: self.pkgConfig(for: systemLib).cFlags)
                 // Add the path to the module map.
                 arguments += ["-I", systemLib.moduleMapPath.parentDirectory.pathString]
             }