Skip to content

Commit 50ac705

Browse files
modocachemodocache
committed
[Tests] Add functional lit tests for OS X
Adds a test runner (lit) that compares annotations in the source code of an XCTest file to actual output when that source code is compiled and run. This acts as a regression test suite for the project. Adds two new Xcode targets: 1. SingleFailingTestCase: An executable that runs `XCTMain()` with a failing test case. It is built and installed to the Tests/Fixtures/Products/ directory. 2. SwiftXCTestTests: An aggregate target that builds SingleFailingTestCase (and, in the future, all test fixtures), then runs the lit test runner, passing the path the `FileCheck`, another LLVM tool that is used to test the output.
1 parent af0aa6d commit 50ac705

File tree

4 files changed

+253
-1
lines changed

4 files changed

+253
-1
lines changed

‎.gitignore

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
.DS_Store
2-
xcuserdata
2+
xcuserdata
3+
4+
# Functional test suite places compiled executables in the source tree
5+
Tests/Functional/Products/*

‎Tests/Functional/Sources/SingleFailingTestCase/main.swift

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// RUN: %S/../../Products/SingleFailingTestCase > %t.out || true
2+
// RUN: FileCheck %s < %t.out
3+
// RUN: rm %t.out
4+
5+
// CHECK: Test Case 'SingleFailingTestCase.test_fails' started.
6+
// CHECK: {{.*}}/Tests/Functional/Sources/SingleFailingTestCase/main.swift:25: error: SingleFailingTestCase.test_fails :
7+
// CHECK: Test Case 'SingleFailingTestCase.test_fails' failed ({{[0-9]+\.[0-9]+}} seconds).
8+
// CHECK: Executed 1 test, with 1 failure (0 unexpected) in {{[0-9]+\.[0-9]+}} ({{[0-9]+\.[0-9]+}}) seconds
9+
// CHECK: Total executed 1 test, with 1 failure (0 unexpected) in {{[0-9]+\.[0-9]+}} ({{[0-9]+\.[0-9]+}}) seconds
10+
11+
#if os(Linux)
12+
import XCTest
13+
#else
14+
import SwiftXCTest
15+
#endif
16+
17+
class SingleFailingTestCase: XCTestCase {
18+
var allTests: [(String, () -> ())] {
19+
return [
20+
("test_fails", test_fails),
21+
]
22+
}
23+
24+
func test_fails() {
25+
XCTAssert(false)
26+
}
27+
}
28+
29+
XCTMain([SingleFailingTestCase()])

‎Tests/lit.cfg

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import lit.formats
2+
import os
3+
4+
config.name = 'SwiftXCTestFunctionalTests'
5+
config.test_format = lit.formats.ShTest(execute_external=False)
6+
config.suffixes = ['.swift']
7+
8+
native_llvm_tools_path = lit_config.params.get('native_llvm_tools_path')
9+
if native_llvm_tools_path is not None:
10+
config.environment['PATH'] = os.path.pathsep.join((
11+
native_llvm_tools_path, config.environment['PATH']))
12+
13+
# vim: set syntax=python:

‎XCTest.xcodeproj/project.pbxproj

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,58 @@
66
objectVersion = 46;
77
objects = {
88

9+
/* Begin PBXAggregateTarget section */
10+
DA075A741C14DAAF000140C8 /* SwiftXCTestTests */ = {
11+
isa = PBXAggregateTarget;
12+
buildConfigurationList = DA075A751C14DAAF000140C8 /* Build configuration list for PBXAggregateTarget "SwiftXCTestTests" */;
13+
buildPhases = (
14+
DA075A7A1C14DABA000140C8 /* ShellScript */,
15+
);
16+
dependencies = (
17+
DA075A7C1C14DAD4000140C8 /* PBXTargetDependency */,
18+
DA075A791C14DAB6000140C8 /* PBXTargetDependency */,
19+
);
20+
name = SwiftXCTestTests;
21+
productName = SwiftXCTestTests;
22+
};
23+
/* End PBXAggregateTarget section */
24+
925
/* Begin PBXBuildFile section */
26+
DAA28A0B1C16017A00DCBAB2 /* SwiftXCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */; };
27+
DAD00B911C1AA434004A84A5 /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = DAD00B8C1C1AA3B3004A84A5 /* main.swift */; };
1028
EA6E86BB1BDEA7DE007C0323 /* XCTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = EA6E86BA1BDEA7DE007C0323 /* XCTest.swift */; };
1129
/* End PBXBuildFile section */
1230

31+
/* Begin PBXContainerItemProxy section */
32+
DA075A781C14DAB6000140C8 /* PBXContainerItemProxy */ = {
33+
isa = PBXContainerItemProxy;
34+
containerPortal = 5B5D86D21BBC74AD00234F36 /* Project object */;
35+
proxyType = 1;
36+
remoteGlobalIDString = DAA28A001C16010900DCBAB2;
37+
remoteInfo = SingleFailingTestCase;
38+
};
39+
DA075A7B1C14DAD4000140C8 /* PBXContainerItemProxy */ = {
40+
isa = PBXContainerItemProxy;
41+
containerPortal = 5B5D86D21BBC74AD00234F36 /* Project object */;
42+
proxyType = 1;
43+
remoteGlobalIDString = 5B5D86DA1BBC74AD00234F36;
44+
remoteInfo = SwiftXCTest;
45+
};
46+
DAA28A091C16017700DCBAB2 /* PBXContainerItemProxy */ = {
47+
isa = PBXContainerItemProxy;
48+
containerPortal = 5B5D86D21BBC74AD00234F36 /* Project object */;
49+
proxyType = 1;
50+
remoteGlobalIDString = 5B5D86DA1BBC74AD00234F36;
51+
remoteInfo = SwiftXCTest;
52+
};
53+
/* End PBXContainerItemProxy section */
54+
1355
/* Begin PBXFileReference section */
1456
5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = SwiftXCTest.framework; sourceTree = BUILT_PRODUCTS_DIR; };
57+
DA075A631C14B8EA000140C8 /* SingleFailingTestCase */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = SingleFailingTestCase; sourceTree = BUILT_PRODUCTS_DIR; };
58+
DAA28A011C16010900DCBAB2 /* SingleFailingTestCase */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = SingleFailingTestCase; sourceTree = BUILT_PRODUCTS_DIR; };
59+
DAD00B8C1C1AA3B3004A84A5 /* main.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = main.swift; sourceTree = "<group>"; };
60+
DAD00B8F1C1AA3B3004A84A5 /* lit.cfg */ = {isa = PBXFileReference; lastKnownFileType = text; path = lit.cfg; sourceTree = "<group>"; };
1561
EA3E74BB1BF2B6D500635A73 /* build_script.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = build_script.py; sourceTree = "<group>"; };
1662
EA6E86BA1BDEA7DE007C0323 /* XCTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTest.swift; sourceTree = "<group>"; };
1763
/* End PBXFileReference section */
@@ -24,12 +70,21 @@
2470
);
2571
runOnlyForDeploymentPostprocessing = 0;
2672
};
73+
DAA289FE1C16010900DCBAB2 /* Frameworks */ = {
74+
isa = PBXFrameworksBuildPhase;
75+
buildActionMask = 2147483647;
76+
files = (
77+
DAA28A0B1C16017A00DCBAB2 /* SwiftXCTest.framework in Frameworks */,
78+
);
79+
runOnlyForDeploymentPostprocessing = 0;
80+
};
2781
/* End PBXFrameworksBuildPhase section */
2882

2983
/* Begin PBXGroup section */
3084
5B5D86D11BBC74AD00234F36 = {
3185
isa = PBXGroup;
3286
children = (
87+
DAD00B831C1AA3B3004A84A5 /* Tests */,
3388
5B5D86DD1BBC74AD00234F36 /* XCTest */,
3489
5B5D86DC1BBC74AD00234F36 /* Products */,
3590
EA3E74BC1BF2B6D700635A73 /* Linux Build */,
@@ -42,6 +97,8 @@
4297
isa = PBXGroup;
4398
children = (
4499
5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */,
100+
DA075A631C14B8EA000140C8 /* SingleFailingTestCase */,
101+
DAA28A011C16010900DCBAB2 /* SingleFailingTestCase */,
45102
);
46103
name = Products;
47104
sourceTree = "<group>";
@@ -54,6 +111,39 @@
54111
path = XCTest;
55112
sourceTree = "<group>";
56113
};
114+
DAD00B831C1AA3B3004A84A5 /* Tests */ = {
115+
isa = PBXGroup;
116+
children = (
117+
DAD00B871C1AA3B3004A84A5 /* Functional */,
118+
DAD00B8F1C1AA3B3004A84A5 /* lit.cfg */,
119+
);
120+
path = Tests;
121+
sourceTree = "<group>";
122+
};
123+
DAD00B871C1AA3B3004A84A5 /* Functional */ = {
124+
isa = PBXGroup;
125+
children = (
126+
DAD00B8A1C1AA3B3004A84A5 /* Sources */,
127+
);
128+
path = Functional;
129+
sourceTree = "<group>";
130+
};
131+
DAD00B8A1C1AA3B3004A84A5 /* Sources */ = {
132+
isa = PBXGroup;
133+
children = (
134+
DAD00B8B1C1AA3B3004A84A5 /* SingleFailingTestCase */,
135+
);
136+
path = Sources;
137+
sourceTree = "<group>";
138+
};
139+
DAD00B8B1C1AA3B3004A84A5 /* SingleFailingTestCase */ = {
140+
isa = PBXGroup;
141+
children = (
142+
DAD00B8C1C1AA3B3004A84A5 /* main.swift */,
143+
);
144+
path = SingleFailingTestCase;
145+
sourceTree = "<group>";
146+
};
57147
EA3E74BC1BF2B6D700635A73 /* Linux Build */ = {
58148
isa = PBXGroup;
59149
children = (
@@ -93,6 +183,23 @@
93183
productReference = 5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */;
94184
productType = "com.apple.product-type.framework";
95185
};
186+
DAA28A001C16010900DCBAB2 /* SingleFailingTestCase */ = {
187+
isa = PBXNativeTarget;
188+
buildConfigurationList = DAA28A071C16010900DCBAB2 /* Build configuration list for PBXNativeTarget "SingleFailingTestCase" */;
189+
buildPhases = (
190+
DAA289FD1C16010900DCBAB2 /* Sources */,
191+
DAA289FE1C16010900DCBAB2 /* Frameworks */,
192+
);
193+
buildRules = (
194+
);
195+
dependencies = (
196+
DAA28A0A1C16017700DCBAB2 /* PBXTargetDependency */,
197+
);
198+
name = SingleFailingTestCase;
199+
productName = SingleFailingTestCase;
200+
productReference = DAA28A011C16010900DCBAB2 /* SingleFailingTestCase */;
201+
productType = "com.apple.product-type.tool";
202+
};
96203
/* End PBXNativeTarget section */
97204

98205
/* Begin PBXProject section */
@@ -122,6 +229,8 @@
122229
projectRoot = "";
123230
targets = (
124231
5B5D86DA1BBC74AD00234F36 /* SwiftXCTest */,
232+
DAA28A001C16010900DCBAB2 /* SingleFailingTestCase */,
233+
DA075A741C14DAAF000140C8 /* SwiftXCTestTests */,
125234
);
126235
};
127236
/* End PBXProject section */
@@ -136,6 +245,22 @@
136245
};
137246
/* End PBXResourcesBuildPhase section */
138247

248+
/* Begin PBXShellScriptBuildPhase section */
249+
DA075A7A1C14DABA000140C8 /* ShellScript */ = {
250+
isa = PBXShellScriptBuildPhase;
251+
buildActionMask = 2147483647;
252+
files = (
253+
);
254+
inputPaths = (
255+
);
256+
outputPaths = (
257+
);
258+
runOnlyForDeploymentPostprocessing = 0;
259+
shellPath = /bin/sh;
260+
shellScript = "lit -v $SRCROOT/Tests --param native_llvm_tools_path=\"$SRCROOT/../build/Ninja-DebugAssert/llvm-macosx-x86_64/bin\"";
261+
};
262+
/* End PBXShellScriptBuildPhase section */
263+
139264
/* Begin PBXSourcesBuildPhase section */
140265
5B5D86D61BBC74AD00234F36 /* Sources */ = {
141266
isa = PBXSourcesBuildPhase;
@@ -145,8 +270,34 @@
145270
);
146271
runOnlyForDeploymentPostprocessing = 0;
147272
};
273+
DAA289FD1C16010900DCBAB2 /* Sources */ = {
274+
isa = PBXSourcesBuildPhase;
275+
buildActionMask = 2147483647;
276+
files = (
277+
DAD00B911C1AA434004A84A5 /* main.swift in Sources */,
278+
);
279+
runOnlyForDeploymentPostprocessing = 0;
280+
};
148281
/* End PBXSourcesBuildPhase section */
149282

283+
/* Begin PBXTargetDependency section */
284+
DA075A791C14DAB6000140C8 /* PBXTargetDependency */ = {
285+
isa = PBXTargetDependency;
286+
target = DAA28A001C16010900DCBAB2 /* SingleFailingTestCase */;
287+
targetProxy = DA075A781C14DAB6000140C8 /* PBXContainerItemProxy */;
288+
};
289+
DA075A7C1C14DAD4000140C8 /* PBXTargetDependency */ = {
290+
isa = PBXTargetDependency;
291+
target = 5B5D86DA1BBC74AD00234F36 /* SwiftXCTest */;
292+
targetProxy = DA075A7B1C14DAD4000140C8 /* PBXContainerItemProxy */;
293+
};
294+
DAA28A0A1C16017700DCBAB2 /* PBXTargetDependency */ = {
295+
isa = PBXTargetDependency;
296+
target = 5B5D86DA1BBC74AD00234F36 /* SwiftXCTest */;
297+
targetProxy = DAA28A091C16017700DCBAB2 /* PBXContainerItemProxy */;
298+
};
299+
/* End PBXTargetDependency section */
300+
150301
/* Begin XCBuildConfiguration section */
151302
5B5D86E11BBC74AD00234F36 /* Debug */ = {
152303
isa = XCBuildConfiguration;
@@ -271,6 +422,44 @@
271422
};
272423
name = Release;
273424
};
425+
DA075A761C14DAAF000140C8 /* Debug */ = {
426+
isa = XCBuildConfiguration;
427+
buildSettings = {
428+
PRODUCT_NAME = "$(TARGET_NAME)";
429+
};
430+
name = Debug;
431+
};
432+
DA075A771C14DAAF000140C8 /* Release */ = {
433+
isa = XCBuildConfiguration;
434+
buildSettings = {
435+
PRODUCT_NAME = "$(TARGET_NAME)";
436+
};
437+
name = Release;
438+
};
439+
DAA28A051C16010900DCBAB2 /* Debug */ = {
440+
isa = XCBuildConfiguration;
441+
buildSettings = {
442+
CODE_SIGN_IDENTITY = "-";
443+
DEPLOYMENT_LOCATION = YES;
444+
DSTROOT = "$(SRCROOT)/Tests/Functional/Products/";
445+
INSTALL_PATH = /;
446+
LD_RUNPATH_SEARCH_PATHS = "$(BUILT_PRODUCTS_DIR)";
447+
PRODUCT_NAME = "$(TARGET_NAME)";
448+
};
449+
name = Debug;
450+
};
451+
DAA28A061C16010900DCBAB2 /* Release */ = {
452+
isa = XCBuildConfiguration;
453+
buildSettings = {
454+
CODE_SIGN_IDENTITY = "-";
455+
DEPLOYMENT_LOCATION = YES;
456+
DSTROOT = "$(SRCROOT)/Tests/Functional/Products/";
457+
INSTALL_PATH = /;
458+
LD_RUNPATH_SEARCH_PATHS = "$(BUILT_PRODUCTS_DIR)";
459+
PRODUCT_NAME = "$(TARGET_NAME)";
460+
};
461+
name = Release;
462+
};
274463
/* End XCBuildConfiguration section */
275464

276465
/* Begin XCConfigurationList section */
@@ -292,6 +481,24 @@
292481
defaultConfigurationIsVisible = 0;
293482
defaultConfigurationName = Release;
294483
};
484+
DA075A751C14DAAF000140C8 /* Build configuration list for PBXAggregateTarget "SwiftXCTestTests" */ = {
485+
isa = XCConfigurationList;
486+
buildConfigurations = (
487+
DA075A761C14DAAF000140C8 /* Debug */,
488+
DA075A771C14DAAF000140C8 /* Release */,
489+
);
490+
defaultConfigurationIsVisible = 0;
491+
defaultConfigurationName = Release;
492+
};
493+
DAA28A071C16010900DCBAB2 /* Build configuration list for PBXNativeTarget "SingleFailingTestCase" */ = {
494+
isa = XCConfigurationList;
495+
buildConfigurations = (
496+
DAA28A051C16010900DCBAB2 /* Debug */,
497+
DAA28A061C16010900DCBAB2 /* Release */,
498+
);
499+
defaultConfigurationIsVisible = 0;
500+
defaultConfigurationName = Release;
501+
};
295502
/* End XCConfigurationList section */
296503
};
297504
rootObject = 5B5D86D21BBC74AD00234F36 /* Project object */;

0 commit comments

Comments
 (0)