Skip to content

Commit b0f725a

Browse files
compnerdcompnerd
authored
Merge pull request #993 from compnerd/tokenization
Driver: add support for tokenizing a Windows-style rsp
2 parents ab32dd4 + 4edd105 commit b0f725a

File tree

2 files changed

+128
-2
lines changed

2 files changed

+128
-2
lines changed

‎Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -968,15 +968,113 @@ extension Driver {
968968
return tokens
969969
}
970970

971+
// https://docs.microsoft.com/en-us/previous-versions//17w5ykft(v=vs.85)?redirectedfrom=MSDN
972+
private static func tokenizeWindowsResponseFile(_ content: String) -> [String] {
973+
let whitespace: [Character] = [" ", "\t", "\r", "\n", "\0" ]
974+
975+
var content = content
976+
var tokens: [String] = []
977+
var token: String = ""
978+
var quoted: Bool = false
979+
980+
while !content.isEmpty {
981+
// Eat whitespace at the beginning
982+
if token.isEmpty {
983+
if let end = content.firstIndex(where: { !whitespace.contains($0) }) {
984+
let count = content.distance(from: content.startIndex, to: end)
985+
content.removeFirst(count)
986+
}
987+
988+
// Stop if this was trailing whitespace.
989+
if content.isEmpty { break }
990+
}
991+
992+
// Treat whitespace, double quotes, and backslashes as special characters.
993+
if let next = content.firstIndex(where: { (quoted ? ["\\", "\""] : [" ", "\t", "\r", "\n", "\0", "\\", "\""]).contains($0) }) {
994+
let count = content.distance(from: content.startIndex, to: next)
995+
token.append(contentsOf: content[..<next])
996+
content.removeFirst(count)
997+
998+
switch content.first {
999+
case " ", "\t", "\r", "\n", "\0":
1000+
tokens.append(token)
1001+
token = ""
1002+
content.removeFirst(1)
1003+
1004+
case "\\":
1005+
// Backslashes are interpreted in a special manner due to use as both
1006+
// a path separator and an escape character. Consume runs of
1007+
// backslashes and following double quote if escaped.
1008+
//
1009+
// - If an even number of backslashes is followed by a double quote,
1010+
// one backslash is emitted for each pair, and the last double quote
1011+
// remains unconsumed. The quote will be processed as the start or
1012+
// end of a quoted string by the tokenizer.
1013+
//
1014+
// - If an odd number of backslashes is followed by a double quote,
1015+
// one backslash is emitted for each pair, and a double quote is
1016+
// emitted for the trailing backslash and quote pair. The double
1017+
// quote is consumed.
1018+
//
1019+
// - Otherwise, backslashes are treated literally.
1020+
if let next = content.firstIndex(where: { $0 != "\\" }) {
1021+
let count = content.distance(from: content.startIndex, to: next)
1022+
if content[next] == "\"" {
1023+
token.append(String(repeating: "\\", count: count / 2))
1024+
content.removeFirst(count)
1025+
1026+
if count % 2 != 0 {
1027+
token.append("\"")
1028+
content.removeFirst(1)
1029+
}
1030+
} else {
1031+
token.append(String(repeating: "\\", count: count))
1032+
content.removeFirst(count)
1033+
}
1034+
} else {
1035+
token.append(String(repeating: "\\", count: content.count))
1036+
content.removeFirst(content.count)
1037+
}
1038+
1039+
case "\"":
1040+
content.removeFirst(1)
1041+
1042+
if quoted, content.first == "\"" {
1043+
// Consequtive double quotes inside a quoted string imples one quote
1044+
token.append("\"")
1045+
content.removeFirst(1)
1046+
}
1047+
1048+
quoted.toggle()
1049+
1050+
default:
1051+
fatalError("unexpected character '\(content.first!)'")
1052+
}
1053+
} else {
1054+
// Consume to end of content.
1055+
token.append(content)
1056+
content.removeFirst(content.count)
1057+
break
1058+
}
1059+
}
1060+
1061+
if !token.isEmpty { tokens.append(token) }
1062+
return tokens.filter { !$0.isEmpty }
1063+
}
1064+
9711065
/// Tokenize each line of the response file, omitting empty lines.
9721066
///
9731067
/// - Parameter content: response file's content to be tokenized.
9741068
private static func tokenizeResponseFile(_ content: String) -> [String] {
975-
#if !canImport(Darwin) && !os(Linux) && !os(Android) && !os(OpenBSD)
1069+
#if !canImport(Darwin) && !os(Linux) && !os(Android) && !os(OpenBSD) && !os(Windows)
9761070
#warning("Response file tokenization unimplemented for platform; behavior may be incorrect")
9771071
#endif
1072+
#if os(Windows)
1073+
return tokenizeWindowsResponseFile(content)
1074+
#else
9781075
return content.split { $0 == "\n" || $0 == "\r\n" }
979-
.flatMap { tokenizeResponseFileLine($0) }
1076+
.flatMap { tokenizeResponseFileLine($0) }
1077+
#endif
9801078
}
9811079

9821080
/// Resolves the absolute path for a response file.

‎Tests/SwiftDriverTests/SwiftDriverTests.swift

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1046,7 +1046,11 @@ final class SwiftDriverTests: XCTestCase {
10461046
let fooPath = path.appending(component: "foo.rsp")
10471047
let barPath = path.appending(component: "bar.rsp")
10481048
try localFileSystem.writeFileContents(fooPath) {
1049+
#if os(Windows)
1050+
$0 <<< "hello\nbye\n\"bye to you\"\n@\(barPath.pathString.nativePathString().escaped())"
1051+
#else
10491052
$0 <<< "hello\nbye\nbye\\ to\\ you\n@\(barPath.pathString.nativePathString().escaped())"
1053+
#endif
10501054
}
10511055
try localFileSystem.writeFileContents(barPath) {
10521056
$0 <<< "from\nbar\n@\(fooPath.pathString.nativePathString().escaped())"
@@ -1069,7 +1073,11 @@ final class SwiftDriverTests: XCTestCase {
10691073
let fooPath = path.appending(component: "foo.rsp")
10701074
let barPath = path.appending(component: "bar.rsp")
10711075
try localFileSystem.writeFileContents(fooPath) {
1076+
#if os(Windows)
1077+
$0 <<< "hello\nbye\n\"bye to you\"\n@bar.rsp"
1078+
#else
10721079
$0 <<< "hello\nbye\nbye\\ to\\ you\n@bar.rsp"
1080+
#endif
10731081
}
10741082
try localFileSystem.writeFileContents(barPath) {
10751083
$0 <<< "from\nbar\n@foo.rsp"
@@ -1095,7 +1103,11 @@ final class SwiftDriverTests: XCTestCase {
10951103
let fooPath = path.appending(components: "subdir", "foo.rsp")
10961104
let barPath = path.appending(components: "subdir", "bar.rsp")
10971105
try localFileSystem.writeFileContents(fooPath) {
1106+
#if os(Windows)
1107+
$0 <<< "hello\nbye\n\"bye to you\"\n@subdir/bar.rsp"
1108+
#else
10981109
$0 <<< "hello\nbye\nbye\\ to\\ you\n@subdir/bar.rsp"
1110+
#endif
10991111
}
11001112
try localFileSystem.writeFileContents(barPath) {
11011113
$0 <<< "from\nbar\n@subdir/foo.rsp"
@@ -1115,6 +1127,21 @@ final class SwiftDriverTests: XCTestCase {
11151127
let barPath = path.appending(component: "bar.rsp")
11161128
let escapingPath = path.appending(component: "escaping.rsp")
11171129

1130+
#if os(Windows)
1131+
try localFileSystem.writeFileContents(fooPath) {
1132+
$0 <<< "a\\b c\\\\d e\\\\\"f g\" h\\\"i j\\\\\\\"k \"lmn\" o pqr \"st \\\"u\" \\v"
1133+
<<< "\n@\(barPath.pathString.nativePathString().escaped())"
1134+
}
1135+
try localFileSystem.writeFileContents(barPath) {
1136+
$0 <<< #"""
1137+
-Xswiftc -use-ld=lld
1138+
-Xcc -IS:\Library\sqlite-3.36.0\usr\include
1139+
-Xlinker -LS:\Library\sqlite-3.36.0\usr\lib
1140+
"""#
1141+
}
1142+
let args = try Driver.expandResponseFiles(["@\(fooPath.pathString)"], fileSystem: localFileSystem, diagnosticsEngine: diags)
1143+
XCTAssertEqual(args, ["a\\b", "c\\\\d", "e\\f g", "h\"i", "j\\\"k", "lmn", "o", "pqr", "st \"u", "\\v", "-Xswiftc", "-use-ld=lld", "-Xcc", "-IS:\\Library\\sqlite-3.36.0\\usr\\include", "-Xlinker", "-LS:\\Library\\sqlite-3.36.0\\usr\\lib"])
1144+
#else
11181145
try localFileSystem.writeFileContents(fooPath) {
11191146
$0 <<< #"""
11201147
Command1 --kkc
@@ -1149,6 +1176,7 @@ final class SwiftDriverTests: XCTestCase {
11491176
XCTAssertEqual(args, ["Command1", "--kkc", "but", "this", "is", #"\\a"#, "command", #"swift"#, "rocks!" ,"compiler", "-Xlinker", "@loader_path", "mkdir", "Quoted Dir", "cd", "Unquoted Dir", "@NotAFile", #"-flag=quoted string with a "quote" inside"#, "-another-flag", "this", "line", "has", "lots", "of", "whitespace"])
11501177
let escapingArgs = try Driver.expandResponseFiles(["@" + escapingPath.pathString], fileSystem: localFileSystem, diagnosticsEngine: diags)
11511178
XCTAssertEqual(escapingArgs, ["swift", "--driver-mode=swiftc", "-v","the end"])
1179+
#endif
11521180
}
11531181
}
11541182

0 commit comments

Comments
 (0)