Skip to content

Commit 4edd105

Browse files
compnerdcompnerd
committed
Driver: add support for tokenizing a Windows-style rsp
This adds support for handling Windows style response files. These are important to get correct as they are consumed by both swift and clang. The differences will result in the `\` path separator getting discarded, yielding invalid paths.
1 parent 00d13d2 commit 4edd105

File tree

2 files changed

+128
-2
lines changed

2 files changed

+128
-2
lines changed

‎Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -968,15 +968,113 @@ extension Driver {
968968
return tokens
969969
}
970970

971+
// https://docs.microsoft.com/en-us/previous-versions//17w5ykft(v=vs.85)?redirectedfrom=MSDN
972+
private static func tokenizeWindowsResponseFile(_ content: String) -> [String] {
973+
let whitespace: [Character] = [" ", "\t", "\r", "\n", "\0" ]
974+
975+
var content = content
976+
var tokens: [String] = []
977+
var token: String = ""
978+
var quoted: Bool = false
979+
980+
while !content.isEmpty {
981+
// Eat whitespace at the beginning
982+
if token.isEmpty {
983+
if let end = content.firstIndex(where: { !whitespace.contains($0) }) {
984+
let count = content.distance(from: content.startIndex, to: end)
985+
content.removeFirst(count)
986+
}
987+
988+
// Stop if this was trailing whitespace.
989+
if content.isEmpty { break }
990+
}
991+
992+
// Treat whitespace, double quotes, and backslashes as special characters.
993+
if let next = content.firstIndex(where: { (quoted ? ["\\", "\""] : [" ", "\t", "\r", "\n", "\0", "\\", "\""]).contains($0) }) {
994+
let count = content.distance(from: content.startIndex, to: next)
995+
token.append(contentsOf: content[..<next])
996+
content.removeFirst(count)
997+
998+
switch content.first {
999+
case " ", "\t", "\r", "\n", "\0":
1000+
tokens.append(token)
1001+
token = ""
1002+
content.removeFirst(1)
1003+
1004+
case "\\":
1005+
// Backslashes are interpreted in a special manner due to use as both
1006+
// a path separator and an escape character. Consume runs of
1007+
// backslashes and following double quote if escaped.
1008+
//
1009+
// - If an even number of backslashes is followed by a double quote,
1010+
// one backslash is emitted for each pair, and the last double quote
1011+
// remains unconsumed. The quote will be processed as the start or
1012+
// end of a quoted string by the tokenizer.
1013+
//
1014+
// - If an odd number of backslashes is followed by a double quote,
1015+
// one backslash is emitted for each pair, and a double quote is
1016+
// emitted for the trailing backslash and quote pair. The double
1017+
// quote is consumed.
1018+
//
1019+
// - Otherwise, backslashes are treated literally.
1020+
if let next = content.firstIndex(where: { $0 != "\\" }) {
1021+
let count = content.distance(from: content.startIndex, to: next)
1022+
if content[next] == "\"" {
1023+
token.append(String(repeating: "\\", count: count / 2))
1024+
content.removeFirst(count)
1025+
1026+
if count % 2 != 0 {
1027+
token.append("\"")
1028+
content.removeFirst(1)
1029+
}
1030+
} else {
1031+
token.append(String(repeating: "\\", count: count))
1032+
content.removeFirst(count)
1033+
}
1034+
} else {
1035+
token.append(String(repeating: "\\", count: content.count))
1036+
content.removeFirst(content.count)
1037+
}
1038+
1039+
case "\"":
1040+
content.removeFirst(1)
1041+
1042+
if quoted, content.first == "\"" {
1043+
// Consequtive double quotes inside a quoted string imples one quote
1044+
token.append("\"")
1045+
content.removeFirst(1)
1046+
}
1047+
1048+
quoted.toggle()
1049+
1050+
default:
1051+
fatalError("unexpected character '\(content.first!)'")
1052+
}
1053+
} else {
1054+
// Consume to end of content.
1055+
token.append(content)
1056+
content.removeFirst(content.count)
1057+
break
1058+
}
1059+
}
1060+
1061+
if !token.isEmpty { tokens.append(token) }
1062+
return tokens.filter { !$0.isEmpty }
1063+
}
1064+
9711065
/// Tokenize each line of the response file, omitting empty lines.
9721066
///
9731067
/// - Parameter content: response file's content to be tokenized.
9741068
private static func tokenizeResponseFile(_ content: String) -> [String] {
975-
#if !canImport(Darwin) && !os(Linux) && !os(Android) && !os(OpenBSD)
1069+
#if !canImport(Darwin) && !os(Linux) && !os(Android) && !os(OpenBSD) && !os(Windows)
9761070
#warning("Response file tokenization unimplemented for platform; behavior may be incorrect")
9771071
#endif
1072+
#if os(Windows)
1073+
return tokenizeWindowsResponseFile(content)
1074+
#else
9781075
return content.split { $0 == "\n" || $0 == "\r\n" }
979-
.flatMap { tokenizeResponseFileLine($0) }
1076+
.flatMap { tokenizeResponseFileLine($0) }
1077+
#endif
9801078
}
9811079

9821080
/// Resolves the absolute path for a response file.

‎Tests/SwiftDriverTests/SwiftDriverTests.swift

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1046,7 +1046,11 @@ final class SwiftDriverTests: XCTestCase {
10461046
let fooPath = path.appending(component: "foo.rsp")
10471047
let barPath = path.appending(component: "bar.rsp")
10481048
try localFileSystem.writeFileContents(fooPath) {
1049+
#if os(Windows)
1050+
$0 <<< "hello\nbye\n\"bye to you\"\n@\(barPath.pathString.nativePathString().escaped())"
1051+
#else
10491052
$0 <<< "hello\nbye\nbye\\ to\\ you\n@\(barPath.pathString.nativePathString().escaped())"
1053+
#endif
10501054
}
10511055
try localFileSystem.writeFileContents(barPath) {
10521056
$0 <<< "from\nbar\n@\(fooPath.pathString.nativePathString().escaped())"
@@ -1069,7 +1073,11 @@ final class SwiftDriverTests: XCTestCase {
10691073
let fooPath = path.appending(component: "foo.rsp")
10701074
let barPath = path.appending(component: "bar.rsp")
10711075
try localFileSystem.writeFileContents(fooPath) {
1076+
#if os(Windows)
1077+
$0 <<< "hello\nbye\n\"bye to you\"\n@bar.rsp"
1078+
#else
10721079
$0 <<< "hello\nbye\nbye\\ to\\ you\n@bar.rsp"
1080+
#endif
10731081
}
10741082
try localFileSystem.writeFileContents(barPath) {
10751083
$0 <<< "from\nbar\n@foo.rsp"
@@ -1095,7 +1103,11 @@ final class SwiftDriverTests: XCTestCase {
10951103
let fooPath = path.appending(components: "subdir", "foo.rsp")
10961104
let barPath = path.appending(components: "subdir", "bar.rsp")
10971105
try localFileSystem.writeFileContents(fooPath) {
1106+
#if os(Windows)
1107+
$0 <<< "hello\nbye\n\"bye to you\"\n@subdir/bar.rsp"
1108+
#else
10981109
$0 <<< "hello\nbye\nbye\\ to\\ you\n@subdir/bar.rsp"
1110+
#endif
10991111
}
11001112
try localFileSystem.writeFileContents(barPath) {
11011113
$0 <<< "from\nbar\n@subdir/foo.rsp"
@@ -1115,6 +1127,21 @@ final class SwiftDriverTests: XCTestCase {
11151127
let barPath = path.appending(component: "bar.rsp")
11161128
let escapingPath = path.appending(component: "escaping.rsp")
11171129

1130+
#if os(Windows)
1131+
try localFileSystem.writeFileContents(fooPath) {
1132+
$0 <<< "a\\b c\\\\d e\\\\\"f g\" h\\\"i j\\\\\\\"k \"lmn\" o pqr \"st \\\"u\" \\v"
1133+
<<< "\n@\(barPath.pathString.nativePathString().escaped())"
1134+
}
1135+
try localFileSystem.writeFileContents(barPath) {
1136+
$0 <<< #"""
1137+
-Xswiftc -use-ld=lld
1138+
-Xcc -IS:\Library\sqlite-3.36.0\usr\include
1139+
-Xlinker -LS:\Library\sqlite-3.36.0\usr\lib
1140+
"""#
1141+
}
1142+
let args = try Driver.expandResponseFiles(["@\(fooPath.pathString)"], fileSystem: localFileSystem, diagnosticsEngine: diags)
1143+
XCTAssertEqual(args, ["a\\b", "c\\\\d", "e\\f g", "h\"i", "j\\\"k", "lmn", "o", "pqr", "st \"u", "\\v", "-Xswiftc", "-use-ld=lld", "-Xcc", "-IS:\\Library\\sqlite-3.36.0\\usr\\include", "-Xlinker", "-LS:\\Library\\sqlite-3.36.0\\usr\\lib"])
1144+
#else
11181145
try localFileSystem.writeFileContents(fooPath) {
11191146
$0 <<< #"""
11201147
Command1 --kkc
@@ -1149,6 +1176,7 @@ final class SwiftDriverTests: XCTestCase {
11491176
XCTAssertEqual(args, ["Command1", "--kkc", "but", "this", "is", #"\\a"#, "command", #"swift"#, "rocks!" ,"compiler", "-Xlinker", "@loader_path", "mkdir", "Quoted Dir", "cd", "Unquoted Dir", "@NotAFile", #"-flag=quoted string with a "quote" inside"#, "-another-flag", "this", "line", "has", "lots", "of", "whitespace"])
11501177
let escapingArgs = try Driver.expandResponseFiles(["@" + escapingPath.pathString], fileSystem: localFileSystem, diagnosticsEngine: diags)
11511178
XCTAssertEqual(escapingArgs, ["swift", "--driver-mode=swiftc", "-v","the end"])
1179+
#endif
11521180
}
11531181
}
11541182

0 commit comments

Comments
 (0)