Skip to content

Commit 61fbdd1

Browse files
modocachemodocache
committed
[Tests] Add functional lit tests for OS X
Adds a test runner (lit) that compares annotations in the source code of an XCTest file to actual output when that source code is compiled and run. This acts as a regression test suite for the project. Adds two new Xcode targets: 1. SingleFailingTestCase: An executable that runs `XCTMain()` with a failing test case. It is built and installed to the Tests/Fixtures/Products/ directory. 2. SwiftXCTestTests: An aggregate target that builds SingleFailingTestCase (and, in the future, all test fixtures), then runs the lit test runner, passing the path the `FileCheck`, another LLVM tool that is used to test the output.
1 parent 0dc69bb commit 61fbdd1

File tree

4 files changed

+253
-1
lines changed

4 files changed

+253
-1
lines changed

‎.gitignore

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
.DS_Store
2-
xcuserdata
2+
xcuserdata
3+
4+
# Functional test suite places compiled executables in the source tree
5+
Tests/Functional/Products/*

‎Tests/Functional/Sources/SingleFailingTestCase/main.swift

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// RUN: %S/../../Products/SingleFailingTestCase > %t.out || true
2+
// RUN: FileCheck %s < %t.out
3+
// RUN: rm %t.out
4+
5+
// CHECK: Test Case 'SingleFailingTestCase.test_fails' started.
6+
// CHECK: {{.*}}/Tests/Functional/Sources/SingleFailingTestCase/main.swift:25: error: SingleFailingTestCase.test_fails :
7+
// CHECK: Test Case 'SingleFailingTestCase.test_fails' failed ({{[0-9]+\.[0-9]+}} seconds).
8+
// CHECK: Executed 1 test, with 1 failure (0 unexpected) in {{[0-9]+\.[0-9]+}} ({{[0-9]+\.[0-9]+}}) seconds
9+
// CHECK: Total executed 1 test, with 1 failure (0 unexpected) in {{[0-9]+\.[0-9]+}} ({{[0-9]+\.[0-9]+}}) seconds
10+
11+
#if os(Linux)
12+
import XCTest
13+
#else
14+
import SwiftXCTest
15+
#endif
16+
17+
class SingleFailingTestCase: XCTestCase {
18+
var allTests: [(String, () -> ())] {
19+
return [
20+
("test_fails", test_fails),
21+
]
22+
}
23+
24+
func test_fails() {
25+
XCTAssert(false)
26+
}
27+
}
28+
29+
XCTMain([SingleFailingTestCase()])

‎Tests/lit.cfg

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import lit.formats
2+
import os
3+
4+
config.name = 'SwiftXCTestFunctionalTests'
5+
config.test_format = lit.formats.ShTest(execute_external=False)
6+
config.suffixes = ['.swift']
7+
8+
native_llvm_tools_path = lit_config.params.get('native_llvm_tools_path')
9+
if native_llvm_tools_path is not None:
10+
config.environment['PATH'] = os.path.pathsep.join((
11+
native_llvm_tools_path, config.environment['PATH']))
12+
13+
# vim: set syntax=python:

‎XCTest.xcodeproj/project.pbxproj

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,64 @@
66
objectVersion = 46;
77
objects = {
88

9+
/* Begin PBXAggregateTarget section */
10+
DA075A741C14DAAF000140C8 /* SwiftXCTestTests */ = {
11+
isa = PBXAggregateTarget;
12+
buildConfigurationList = DA075A751C14DAAF000140C8 /* Build configuration list for PBXAggregateTarget "SwiftXCTestTests" */;
13+
buildPhases = (
14+
DA075A7A1C14DABA000140C8 /* ShellScript */,
15+
);
16+
dependencies = (
17+
DA075A7C1C14DAD4000140C8 /* PBXTargetDependency */,
18+
DA075A791C14DAB6000140C8 /* PBXTargetDependency */,
19+
);
20+
name = SwiftXCTestTests;
21+
productName = SwiftXCTestTests;
22+
};
23+
/* End PBXAggregateTarget section */
24+
925
/* Begin PBXBuildFile section */
1026
B11FFA2A1C1603B6004297C2 /* XCTAssert.swift in Sources */ = {isa = PBXBuildFile; fileRef = B11FFA291C1603B6004297C2 /* XCTAssert.swift */; };
1127
B11FFA2C1C160434004297C2 /* XCTestCaseProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = B11FFA2B1C160434004297C2 /* XCTestCaseProvider.swift */; };
1228
B11FFA2E1C16053C004297C2 /* XCTestCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = B11FFA2D1C16053C004297C2 /* XCTestCase.swift */; };
29+
DAA28A0B1C16017A00DCBAB2 /* SwiftXCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */; };
30+
DAD00B911C1AA434004A84A5 /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = DAD00B8C1C1AA3B3004A84A5 /* main.swift */; };
1331
EA6E86BB1BDEA7DE007C0323 /* XCTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = EA6E86BA1BDEA7DE007C0323 /* XCTest.swift */; };
1432
/* End PBXBuildFile section */
1533

34+
/* Begin PBXContainerItemProxy section */
35+
DA075A781C14DAB6000140C8 /* PBXContainerItemProxy */ = {
36+
isa = PBXContainerItemProxy;
37+
containerPortal = 5B5D86D21BBC74AD00234F36 /* Project object */;
38+
proxyType = 1;
39+
remoteGlobalIDString = DAA28A001C16010900DCBAB2;
40+
remoteInfo = SingleFailingTestCase;
41+
};
42+
DA075A7B1C14DAD4000140C8 /* PBXContainerItemProxy */ = {
43+
isa = PBXContainerItemProxy;
44+
containerPortal = 5B5D86D21BBC74AD00234F36 /* Project object */;
45+
proxyType = 1;
46+
remoteGlobalIDString = 5B5D86DA1BBC74AD00234F36;
47+
remoteInfo = SwiftXCTest;
48+
};
49+
DAA28A091C16017700DCBAB2 /* PBXContainerItemProxy */ = {
50+
isa = PBXContainerItemProxy;
51+
containerPortal = 5B5D86D21BBC74AD00234F36 /* Project object */;
52+
proxyType = 1;
53+
remoteGlobalIDString = 5B5D86DA1BBC74AD00234F36;
54+
remoteInfo = SwiftXCTest;
55+
};
56+
/* End PBXContainerItemProxy section */
57+
1658
/* Begin PBXFileReference section */
1759
5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = SwiftXCTest.framework; sourceTree = BUILT_PRODUCTS_DIR; };
1860
B11FFA291C1603B6004297C2 /* XCTAssert.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTAssert.swift; sourceTree = "<group>"; };
1961
B11FFA2B1C160434004297C2 /* XCTestCaseProvider.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTestCaseProvider.swift; sourceTree = "<group>"; };
2062
B11FFA2D1C16053C004297C2 /* XCTestCase.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTestCase.swift; sourceTree = "<group>"; };
63+
DA075A631C14B8EA000140C8 /* SingleFailingTestCase */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = SingleFailingTestCase; sourceTree = BUILT_PRODUCTS_DIR; };
64+
DAA28A011C16010900DCBAB2 /* SingleFailingTestCase */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = SingleFailingTestCase; sourceTree = BUILT_PRODUCTS_DIR; };
65+
DAD00B8C1C1AA3B3004A84A5 /* main.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = main.swift; sourceTree = "<group>"; };
66+
DAD00B8F1C1AA3B3004A84A5 /* lit.cfg */ = {isa = PBXFileReference; lastKnownFileType = text; path = lit.cfg; sourceTree = "<group>"; };
2167
EA3E74BB1BF2B6D500635A73 /* build_script.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = build_script.py; sourceTree = "<group>"; };
2268
EA6E86BA1BDEA7DE007C0323 /* XCTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTest.swift; sourceTree = "<group>"; };
2369
/* End PBXFileReference section */
@@ -30,13 +76,22 @@
3076
);
3177
runOnlyForDeploymentPostprocessing = 0;
3278
};
79+
DAA289FE1C16010900DCBAB2 /* Frameworks */ = {
80+
isa = PBXFrameworksBuildPhase;
81+
buildActionMask = 2147483647;
82+
files = (
83+
DAA28A0B1C16017A00DCBAB2 /* SwiftXCTest.framework in Frameworks */,
84+
);
85+
runOnlyForDeploymentPostprocessing = 0;
86+
};
3387
/* End PBXFrameworksBuildPhase section */
3488

3589
/* Begin PBXGroup section */
3690
5B5D86D11BBC74AD00234F36 = {
3791
isa = PBXGroup;
3892
children = (
3993
5B5D86DD1BBC74AD00234F36 /* XCTest */,
94+
DAD00B831C1AA3B3004A84A5 /* Tests */,
4095
5B5D86DC1BBC74AD00234F36 /* Products */,
4196
EA3E74BC1BF2B6D700635A73 /* Linux Build */,
4297
);
@@ -48,6 +103,8 @@
48103
isa = PBXGroup;
49104
children = (
50105
5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */,
106+
DA075A631C14B8EA000140C8 /* SingleFailingTestCase */,
107+
DAA28A011C16010900DCBAB2 /* SingleFailingTestCase */,
51108
);
52109
name = Products;
53110
sourceTree = "<group>";
@@ -63,6 +120,39 @@
63120
path = XCTest;
64121
sourceTree = "<group>";
65122
};
123+
DAD00B831C1AA3B3004A84A5 /* Tests */ = {
124+
isa = PBXGroup;
125+
children = (
126+
DAD00B871C1AA3B3004A84A5 /* Functional */,
127+
DAD00B8F1C1AA3B3004A84A5 /* lit.cfg */,
128+
);
129+
path = Tests;
130+
sourceTree = "<group>";
131+
};
132+
DAD00B871C1AA3B3004A84A5 /* Functional */ = {
133+
isa = PBXGroup;
134+
children = (
135+
DAD00B8A1C1AA3B3004A84A5 /* Sources */,
136+
);
137+
path = Functional;
138+
sourceTree = "<group>";
139+
};
140+
DAD00B8A1C1AA3B3004A84A5 /* Sources */ = {
141+
isa = PBXGroup;
142+
children = (
143+
DAD00B8B1C1AA3B3004A84A5 /* SingleFailingTestCase */,
144+
);
145+
path = Sources;
146+
sourceTree = "<group>";
147+
};
148+
DAD00B8B1C1AA3B3004A84A5 /* SingleFailingTestCase */ = {
149+
isa = PBXGroup;
150+
children = (
151+
DAD00B8C1C1AA3B3004A84A5 /* main.swift */,
152+
);
153+
path = SingleFailingTestCase;
154+
sourceTree = "<group>";
155+
};
66156
EA3E74BC1BF2B6D700635A73 /* Linux Build */ = {
67157
isa = PBXGroup;
68158
children = (
@@ -102,6 +192,23 @@
102192
productReference = 5B5D86DB1BBC74AD00234F36 /* SwiftXCTest.framework */;
103193
productType = "com.apple.product-type.framework";
104194
};
195+
DAA28A001C16010900DCBAB2 /* SingleFailingTestCase */ = {
196+
isa = PBXNativeTarget;
197+
buildConfigurationList = DAA28A071C16010900DCBAB2 /* Build configuration list for PBXNativeTarget "SingleFailingTestCase" */;
198+
buildPhases = (
199+
DAA289FD1C16010900DCBAB2 /* Sources */,
200+
DAA289FE1C16010900DCBAB2 /* Frameworks */,
201+
);
202+
buildRules = (
203+
);
204+
dependencies = (
205+
DAA28A0A1C16017700DCBAB2 /* PBXTargetDependency */,
206+
);
207+
name = SingleFailingTestCase;
208+
productName = SingleFailingTestCase;
209+
productReference = DAA28A011C16010900DCBAB2 /* SingleFailingTestCase */;
210+
productType = "com.apple.product-type.tool";
211+
};
105212
/* End PBXNativeTarget section */
106213

107214
/* Begin PBXProject section */
@@ -131,6 +238,8 @@
131238
projectRoot = "";
132239
targets = (
133240
5B5D86DA1BBC74AD00234F36 /* SwiftXCTest */,
241+
DAA28A001C16010900DCBAB2 /* SingleFailingTestCase */,
242+
DA075A741C14DAAF000140C8 /* SwiftXCTestTests */,
134243
);
135244
};
136245
/* End PBXProject section */
@@ -145,6 +254,22 @@
145254
};
146255
/* End PBXResourcesBuildPhase section */
147256

257+
/* Begin PBXShellScriptBuildPhase section */
258+
DA075A7A1C14DABA000140C8 /* ShellScript */ = {
259+
isa = PBXShellScriptBuildPhase;
260+
buildActionMask = 2147483647;
261+
files = (
262+
);
263+
inputPaths = (
264+
);
265+
outputPaths = (
266+
);
267+
runOnlyForDeploymentPostprocessing = 0;
268+
shellPath = /bin/sh;
269+
shellScript = "lit -v $SRCROOT/Tests --param native_llvm_tools_path=\"$SRCROOT/../build/Ninja-DebugAssert/llvm-macosx-x86_64/bin\"";
270+
};
271+
/* End PBXShellScriptBuildPhase section */
272+
148273
/* Begin PBXSourcesBuildPhase section */
149274
5B5D86D61BBC74AD00234F36 /* Sources */ = {
150275
isa = PBXSourcesBuildPhase;
@@ -157,8 +282,34 @@
157282
);
158283
runOnlyForDeploymentPostprocessing = 0;
159284
};
285+
DAA289FD1C16010900DCBAB2 /* Sources */ = {
286+
isa = PBXSourcesBuildPhase;
287+
buildActionMask = 2147483647;
288+
files = (
289+
DAD00B911C1AA434004A84A5 /* main.swift in Sources */,
290+
);
291+
runOnlyForDeploymentPostprocessing = 0;
292+
};
160293
/* End PBXSourcesBuildPhase section */
161294

295+
/* Begin PBXTargetDependency section */
296+
DA075A791C14DAB6000140C8 /* PBXTargetDependency */ = {
297+
isa = PBXTargetDependency;
298+
target = DAA28A001C16010900DCBAB2 /* SingleFailingTestCase */;
299+
targetProxy = DA075A781C14DAB6000140C8 /* PBXContainerItemProxy */;
300+
};
301+
DA075A7C1C14DAD4000140C8 /* PBXTargetDependency */ = {
302+
isa = PBXTargetDependency;
303+
target = 5B5D86DA1BBC74AD00234F36 /* SwiftXCTest */;
304+
targetProxy = DA075A7B1C14DAD4000140C8 /* PBXContainerItemProxy */;
305+
};
306+
DAA28A0A1C16017700DCBAB2 /* PBXTargetDependency */ = {
307+
isa = PBXTargetDependency;
308+
target = 5B5D86DA1BBC74AD00234F36 /* SwiftXCTest */;
309+
targetProxy = DAA28A091C16017700DCBAB2 /* PBXContainerItemProxy */;
310+
};
311+
/* End PBXTargetDependency section */
312+
162313
/* Begin XCBuildConfiguration section */
163314
5B5D86E11BBC74AD00234F36 /* Debug */ = {
164315
isa = XCBuildConfiguration;
@@ -283,6 +434,44 @@
283434
};
284435
name = Release;
285436
};
437+
DA075A761C14DAAF000140C8 /* Debug */ = {
438+
isa = XCBuildConfiguration;
439+
buildSettings = {
440+
PRODUCT_NAME = "$(TARGET_NAME)";
441+
};
442+
name = Debug;
443+
};
444+
DA075A771C14DAAF000140C8 /* Release */ = {
445+
isa = XCBuildConfiguration;
446+
buildSettings = {
447+
PRODUCT_NAME = "$(TARGET_NAME)";
448+
};
449+
name = Release;
450+
};
451+
DAA28A051C16010900DCBAB2 /* Debug */ = {
452+
isa = XCBuildConfiguration;
453+
buildSettings = {
454+
CODE_SIGN_IDENTITY = "-";
455+
DEPLOYMENT_LOCATION = YES;
456+
DSTROOT = "$(SRCROOT)/Tests/Functional/Products/";
457+
INSTALL_PATH = /;
458+
LD_RUNPATH_SEARCH_PATHS = "$(BUILT_PRODUCTS_DIR)";
459+
PRODUCT_NAME = "$(TARGET_NAME)";
460+
};
461+
name = Debug;
462+
};
463+
DAA28A061C16010900DCBAB2 /* Release */ = {
464+
isa = XCBuildConfiguration;
465+
buildSettings = {
466+
CODE_SIGN_IDENTITY = "-";
467+
DEPLOYMENT_LOCATION = YES;
468+
DSTROOT = "$(SRCROOT)/Tests/Functional/Products/";
469+
INSTALL_PATH = /;
470+
LD_RUNPATH_SEARCH_PATHS = "$(BUILT_PRODUCTS_DIR)";
471+
PRODUCT_NAME = "$(TARGET_NAME)";
472+
};
473+
name = Release;
474+
};
286475
/* End XCBuildConfiguration section */
287476

288477
/* Begin XCConfigurationList section */
@@ -304,6 +493,24 @@
304493
defaultConfigurationIsVisible = 0;
305494
defaultConfigurationName = Release;
306495
};
496+
DA075A751C14DAAF000140C8 /* Build configuration list for PBXAggregateTarget "SwiftXCTestTests" */ = {
497+
isa = XCConfigurationList;
498+
buildConfigurations = (
499+
DA075A761C14DAAF000140C8 /* Debug */,
500+
DA075A771C14DAAF000140C8 /* Release */,
501+
);
502+
defaultConfigurationIsVisible = 0;
503+
defaultConfigurationName = Release;
504+
};
505+
DAA28A071C16010900DCBAB2 /* Build configuration list for PBXNativeTarget "SingleFailingTestCase" */ = {
506+
isa = XCConfigurationList;
507+
buildConfigurations = (
508+
DAA28A051C16010900DCBAB2 /* Debug */,
509+
DAA28A061C16010900DCBAB2 /* Release */,
510+
);
511+
defaultConfigurationIsVisible = 0;
512+
defaultConfigurationName = Release;
513+
};
307514
/* End XCConfigurationList section */
308515
};
309516
rootObject = 5B5D86D21BBC74AD00234F36 /* Project object */;

0 commit comments

Comments
 (0)