Skip to content

Commit 5cbca0f

Browse files
spevansspevans
committed
SR-7620: Use charset encoding if available for NSString(contentsOf:usedEncoding:)
1 parent 76e3f8f commit 5cbca0f

File tree

9 files changed

+96
-14
lines changed

9 files changed

+96
-14
lines changed

‎DarwinCompatibilityTests.xcodeproj/project.pbxproj

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
objects = {
88

99
/* Begin PBXBuildFile section */
10+
B907F36F20BB188800013CBE /* NSString-ISO-8859-1-data.txt in Resources */ = {isa = PBXBuildFile; fileRef = B907F36E20BB188800013CBE /* NSString-ISO-8859-1-data.txt */; };
11+
B917D32420B0DB9700728EE0 /* Foundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = B917D32320B0DB9700728EE0 /* Foundation.framework */; };
12+
B917D32620B0DE2000728EE0 /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = B917D32520B0DE2000728EE0 /* main.swift */; };
1013
B95788861F6FB9470003EB01 /* TestNSNumberBridging.swift in Sources */ = {isa = PBXBuildFile; fileRef = B95788851F6FB9470003EB01 /* TestNSNumberBridging.swift */; };
1114
B9C89ED21F6BF67C00087AF4 /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = B9C89ED11F6BF67C00087AF4 /* XCTest.framework */; };
1215
B9C89F361F6BF89C00087AF4 /* TestScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9C89EE61F6BF88F00087AF4 /* TestScanner.swift */; };
@@ -123,6 +126,10 @@
123126
/* End PBXCopyFilesBuildPhase section */
124127

125128
/* Begin PBXFileReference section */
129+
B907F36E20BB188800013CBE /* NSString-ISO-8859-1-data.txt */ = {isa = PBXFileReference; lastKnownFileType = text; name = "NSString-ISO-8859-1-data.txt"; path = "TestFoundation/Resources/NSString-ISO-8859-1-data.txt"; sourceTree = "<group>"; };
130+
B917D31C20B0DB8B00728EE0 /* xdgTestHelper */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = xdgTestHelper; sourceTree = BUILT_PRODUCTS_DIR; };
131+
B917D32320B0DB9700728EE0 /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; };
132+
B917D32520B0DE2000728EE0 /* main.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = main.swift; path = TestFoundation/xdgTestHelper/main.swift; sourceTree = "<group>"; };
126133
B95788851F6FB9470003EB01 /* TestNSNumberBridging.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = TestNSNumberBridging.swift; path = TestFoundation/TestNSNumberBridging.swift; sourceTree = "<group>"; };
127134
B9C89EC11F6BF47D00087AF4 /* DarwinCompatibilityTests */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = DarwinCompatibilityTests; sourceTree = BUILT_PRODUCTS_DIR; };
128135
B9C89ED11F6BF67C00087AF4 /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; };
@@ -269,6 +276,7 @@
269276
B9C89FAA1F6DCAE700087AF4 /* NSString-UTF16-LE-data.txt */,
270277
B9C89FB01F6DCAE900087AF4 /* NSString-UTF32-BE-data.txt */,
271278
B9C89FA51F6DCAE500087AF4 /* NSString-UTF32-LE-data.txt */,
279+
B907F36E20BB188800013CBE /* NSString-ISO-8859-1-data.txt */,
272280
B9C89FAE1F6DCAE800087AF4 /* NSStringTestData.txt */,
273281
B9C89FB21F6DCAE900087AF4 /* NSURLTestData.plist */,
274282
B9C89FB61F6DCAEA00087AF4 /* NSXMLDocumentTestData.xml */,
@@ -467,6 +475,7 @@
467475
isa = PBXResourcesBuildPhase;
468476
buildActionMask = 2147483647;
469477
files = (
478+
B907F36F20BB188800013CBE /* NSString-ISO-8859-1-data.txt in Resources */,
470479
B9C89FBA1F6DCAEB00087AF4 /* NSString-UTF32-LE-data.txt in Resources */,
471480
B9C89FBB1F6DCAEB00087AF4 /* NSKeyedUnarchiver-EdgeInsetsTest.plist in Resources */,
472481
B9C89FBC1F6DCAEB00087AF4 /* NSKeyedUnarchiver-ConcreteValueTest.plist in Resources */,

‎Foundation.xcodeproj/project.pbxproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,7 @@
330330
9F0DD3571ECD783500F68030 /* SwiftFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5B5D885D1BBC938800234F36 /* SwiftFoundation.framework */; };
331331
A058C2021E529CF100B07AA1 /* TestMassFormatter.swift in Sources */ = {isa = PBXBuildFile; fileRef = A058C2011E529CF100B07AA1 /* TestMassFormatter.swift */; };
332332
AE35A1861CBAC85E0042DB84 /* SwiftFoundation.h in Headers */ = {isa = PBXBuildFile; fileRef = AE35A1851CBAC85E0042DB84 /* SwiftFoundation.h */; settings = {ATTRIBUTES = (Public, ); }; };
333+
B907F36B20BB07A700013CBE /* NSString-ISO-8859-1-data.txt in Resources */ = {isa = PBXBuildFile; fileRef = B907F36A20BB07A700013CBE /* NSString-ISO-8859-1-data.txt */; };
333334
B90C57BB1EEEEA5A005208AE /* TestFileManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 525AECEB1BF2C96400D15BB0 /* TestFileManager.swift */; };
334335
B90C57BC1EEEEA5A005208AE /* TestThread.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5E5835F31C20C9B500C81317 /* TestThread.swift */; };
335336
B910957A1EEF237800A71930 /* NSString-UTF16-LE-data.txt in Resources */ = {isa = PBXBuildFile; fileRef = B91095781EEF237800A71930 /* NSString-UTF16-LE-data.txt */; };
@@ -813,6 +814,7 @@
813814
A5A34B551C18C85D00FD972B /* TestByteCountFormatter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestByteCountFormatter.swift; sourceTree = "<group>"; };
814815
AE35A1851CBAC85E0042DB84 /* SwiftFoundation.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SwiftFoundation.h; sourceTree = "<group>"; };
815816
B167A6641ED7303F0040B09A /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = "<group>"; };
817+
B907F36A20BB07A700013CBE /* NSString-ISO-8859-1-data.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "NSString-ISO-8859-1-data.txt"; sourceTree = "<group>"; };
816818
B91095781EEF237800A71930 /* NSString-UTF16-LE-data.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "NSString-UTF16-LE-data.txt"; sourceTree = "<group>"; };
817819
B91095791EEF237800A71930 /* NSString-UTF16-BE-data.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "NSString-UTF16-BE-data.txt"; sourceTree = "<group>"; };
818820
B933A79C1F3055F600FE6846 /* NSString-UTF32-BE-data.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = "NSString-UTF32-BE-data.txt"; sourceTree = "<group>"; };
@@ -1457,6 +1459,7 @@
14571459
B91095791EEF237800A71930 /* NSString-UTF16-BE-data.txt */,
14581460
B933A79C1F3055F600FE6846 /* NSString-UTF32-BE-data.txt */,
14591461
B933A79D1F3055F600FE6846 /* NSString-UTF32-LE-data.txt */,
1462+
B907F36A20BB07A700013CBE /* NSString-ISO-8859-1-data.txt */,
14601463
528776181BF27D9500CB0090 /* Test.plist */,
14611464
EA66F63B1BF1619600136161 /* NSURLTestData.plist */,
14621465
E1A3726E1C31EBFB0023AF4D /* NSXMLDocumentTestData.xml */,
@@ -2150,6 +2153,7 @@
21502153
D3E8D6D51C36AC0C00295652 /* NSKeyedUnarchiver-RectTest.plist in Resources */,
21512154
D3A597F81C3415CC00295652 /* NSKeyedUnarchiver-URLTest.plist in Resources */,
21522155
D3E8D6D31C36982700295652 /* NSKeyedUnarchiver-EdgeInsetsTest.plist in Resources */,
2156+
B907F36B20BB07A700013CBE /* NSString-ISO-8859-1-data.txt in Resources */,
21532157
D370696E1C394FBF00295652 /* NSKeyedUnarchiver-RangeTest.plist in Resources */,
21542158
D3A597F71C3415CC00295652 /* NSKeyedUnarchiver-ArrayTest.plist in Resources */,
21552159
CE19A88C1C23AA2300B4CB6A /* NSStringTestData.txt in Resources */,

‎Foundation/NSData.swift

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -185,31 +185,36 @@ open class NSData : NSObject, NSCopying, NSMutableCopying, NSSecureCoding {
185185
/// Initializes a data object with the data from the location specified by a given URL.
186186
public init(contentsOf url: URL, options readOptionsMask: ReadingOptions = []) throws {
187187
super.init()
188-
try _contentsOf(url: url, options: readOptionsMask)
188+
let (data, _) = try NSData.contentsOf(url: url, options: readOptionsMask)
189+
_init(bytes: UnsafeMutableRawPointer(mutating: data.bytes), length: data.length, copy: true)
189190
}
190191

191192
/// Initializes a data object with the data from the location specified by a given URL.
192193
public init?(contentsOf url: URL) {
193194
super.init()
194195
do {
195-
try _contentsOf(url: url)
196+
let (data, _) = try NSData.contentsOf(url: url)
197+
_init(bytes: UnsafeMutableRawPointer(mutating: data.bytes), length: data.length, copy: true)
196198
} catch {
197199
return nil
198200
}
199201
}
200202

201-
/// Initializes a data object with the data from the location specified by a given URL.
202-
private func _contentsOf(url: URL, options readOptionsMask: ReadingOptions = []) throws {
203+
internal static func contentsOf(url: URL, options readOptionsMask: ReadingOptions = []) throws -> (NSData, URLResponse?) {
204+
let readResult: NSData
205+
var urlResponse: URLResponse?
206+
203207
if url.isFileURL {
204-
let readResult = try NSData.readBytesFromFileWithExtendedAttributes(url.path, options: readOptionsMask)
205-
_init(bytes: readResult.bytes, length: readResult.length, copy: false, deallocator: readResult.deallocator)
208+
let data = try NSData.readBytesFromFileWithExtendedAttributes(url.path, options: readOptionsMask)
209+
readResult = NSData(bytesNoCopy: data.bytes, length: data.length, deallocator: data.deallocator)
206210
} else {
207211
let session = URLSession(configuration: URLSessionConfiguration.default)
208212
let cond = NSCondition()
209213
var resError: Error?
210214
var resData: Data?
211215
let task = session.dataTask(with: url, completionHandler: { data, response, error in
212216
resData = data
217+
urlResponse = response
213218
resError = error
214219
cond.broadcast()
215220
})
@@ -218,8 +223,9 @@ open class NSData : NSObject, NSCopying, NSMutableCopying, NSSecureCoding {
218223
guard let data = resData else {
219224
throw resError!
220225
}
221-
_init(bytes: UnsafeMutableRawPointer(mutating: data._nsObject.bytes), length: data.count, copy: true)
226+
readResult = NSData(bytes: UnsafeMutableRawPointer(mutating: data._nsObject.bytes), length: data.count)
222227
}
228+
return (readResult, urlResponse)
223229
}
224230

225231
/// Initializes a data object with the given Base64 encoded string.

‎Foundation/NSString.swift

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1254,12 +1254,14 @@ extension NSString {
12541254
public convenience init(contentsOfFile path: String, encoding enc: UInt) throws {
12551255
try self.init(contentsOf: URL(fileURLWithPath: path), encoding: enc)
12561256
}
1257-
1257+
12581258
public convenience init(contentsOf url: URL, usedEncoding enc: UnsafeMutablePointer<UInt>?) throws {
1259-
let readResult = try NSData(contentsOf: url, options:[])
1259+
let (readResult, urlResponse) = try NSData.contentsOf(url: url)
12601260

12611261
let encoding: UInt
12621262
let offset: Int
1263+
// Look for a BOM (Byte Order Marker) to try and determine the text Encoding, this also skips
1264+
// over the bytes. This takes precedence over the textEncoding in the http header
12631265
let bytePtr = readResult.bytes.bindMemory(to: UInt8.self, capacity:readResult.length)
12641266
if readResult.length >= 4 && bytePtr[0] == 0xFF && bytePtr[1] == 0xFE && bytePtr[2] == 0x00 && bytePtr[3] == 0x00 {
12651267
encoding = String.Encoding.utf32LittleEndian.rawValue
@@ -1277,14 +1279,15 @@ extension NSString {
12771279
encoding = String.Encoding.utf32BigEndian.rawValue
12781280
offset = 4
12791281
}
1280-
else {
1282+
else if let charSet = urlResponse?.textEncodingName, let textEncoding = String.Encoding(charSet: charSet) {
1283+
encoding = textEncoding.rawValue
1284+
offset = 0
1285+
} else {
12811286
//Need to work on more conditions. This should be the default
12821287
encoding = String.Encoding.utf8.rawValue
12831288
offset = 0
12841289
}
12851290

1286-
enc?.pointee = encoding
1287-
12881291
// Since the encoding being passed includes the byte order the BOM wont be checked or skipped, so pass offset to
12891292
// manually skip the BOM header.
12901293
guard let cf = CFStringCreateWithBytes(kCFAllocatorDefault, bytePtr + offset, readResult.length - offset,
@@ -1301,6 +1304,7 @@ extension NSString {
13011304
"NSDebugDescription" : "Unable to bridge CFString to String."
13021305
])
13031306
}
1307+
enc?.pointee = encoding
13041308
}
13051309

13061310
public convenience init(contentsOfFile path: String, usedEncoding enc: UnsafeMutablePointer<UInt>?) throws {

‎Foundation/StringEncodings.swift

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,39 @@ extension String {
2727
public static let utf32 = Encoding(rawValue: 0x8c000100)
2828
public static let utf32BigEndian = Encoding(rawValue: 0x98000100)
2929
public static let utf32LittleEndian = Encoding(rawValue: 0x9c000100)
30+
31+
// Map selected IANA character set names to encodings, see
32+
// https://www.iana.org/assignments/character-sets/character-sets.xhtml
33+
internal init?(charSet: String) {
34+
let encoding: Encoding?
35+
36+
switch charSet.lowercased() {
37+
case "us-ascii": encoding = .ascii
38+
case "utf-8": encoding = .utf8
39+
case "utf-16": encoding = .utf16
40+
case "utf-16be": encoding = .utf16BigEndian
41+
case "utf-16le": encoding = .utf16LittleEndian
42+
case "utf-32": encoding = .utf32
43+
case "utf-32be": encoding = .utf32BigEndian
44+
case "utf-32le": encoding = .utf32LittleEndian
45+
case "iso-8859-1": encoding = .isoLatin1
46+
case "iso-8859-2": encoding = .isoLatin2
47+
case "iso-2022-jp": encoding = .iso2022JP
48+
case "windows-1250": encoding = .windowsCP1250
49+
case "windows-1251": encoding = .windowsCP1251
50+
case "windows-1252": encoding = .windowsCP1252
51+
case "windows-1253": encoding = .windowsCP1253
52+
case "windows-1254": encoding = .windowsCP1254
53+
case "shift_jis": encoding = .shiftJIS
54+
case "euc-jp": encoding = .japaneseEUC
55+
case "macintosh": encoding = .macOSRoman
56+
default: encoding = nil
57+
}
58+
guard let value = encoding?.rawValue else {
59+
return nil
60+
}
61+
rawValue = value
62+
}
3063
}
3164

3265
public typealias EncodingConversionOptions = NSString.EncodingConversionOptions
@@ -50,6 +83,7 @@ extension String.Encoding : CustomStringConvertible {
5083
}
5184
}
5285

86+
5387
@available(*, unavailable, renamed: "String.Encoding")
5488
public typealias NSStringEncoding = UInt
5589

‎TestFoundation/HTTPServer.swift

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,7 @@ struct _HTTPResponse {
314314
enum Response : Int {
315315
case OK = 200
316316
case REDIRECT = 302
317+
case NOTFOUND = 404
317318
}
318319
private let responseCode: Response
319320
private let headers: String
@@ -366,6 +367,16 @@ public class TestURLSessionServer {
366367
if req.uri.hasPrefix("/LandOfTheLostCities/") {
367368
/* these are all misbehaving servers */
368369
try httpServer.respondWithBrokenResponses(uri: req.uri)
370+
} else if req.uri == "/NSString-ISO-8859-1-data.txt" {
371+
// Serve this directly as binary data to avoid any String encoding conversions.
372+
if let url = testBundle().url(forResource: "NSString-ISO-8859-1-data", withExtension: "txt"),
373+
let content = try? Data(contentsOf: url) {
374+
var responseData = "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=ISO-8859-1\r\nContent-Length: \(content.count)\r\n\r\n".data(using: .ascii)!
375+
responseData.append(content)
376+
try httpServer.socket.writeRawData(responseData)
377+
} else {
378+
try httpServer.respond(with: _HTTPResponse(response: .NOTFOUND, body: "Not Found"))
379+
}
369380
} else {
370381
try httpServer.respond(with: process(request: req), startDelay: self.startDelay, sendDelay: self.sendDelay, bodyChunks: self.bodyChunks)
371382
}

‎TestFoundation/Resources/NSString-ISO-8859-1-data.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
This file is encoded as ISO-8859-1
2+
�������
3+
�

‎TestFoundation/TestNSString.swift

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ internal let kCFStringEncodingUTF32LE = CFStringBuiltInEncodings.UTF32LE.rawVal
3636
#endif
3737

3838

39-
class TestNSString : XCTestCase {
39+
class TestNSString: LoopbackServerTest {
4040

4141
static var allTests: [(String, (TestNSString) -> () throws -> Void)] {
4242
return [
@@ -301,6 +301,16 @@ class TestNSString : XCTestCase {
301301
} catch {
302302
XCTFail("Unable to init NSString from contentsOf:encoding:")
303303
}
304+
305+
let url = URL(string: "http://127.0.0.1:\(TestURLSession.serverPort)/NSString-ISO-8859-1-data.txt")!
306+
var enc: UInt = 0
307+
let contents = try? NSString(contentsOf: url, usedEncoding: &enc)
308+
309+
XCTAssertNotNil(contents)
310+
XCTAssertEqual(enc, String.Encoding.isoLatin1.rawValue)
311+
if let contents = contents {
312+
XCTAssertEqual(contents, "This file is encoded as ISO-8859-1\nÀÁÂÃÄÅÿ\n±\n")
313+
}
304314
}
305315

306316
func test_FromContentOfFileUsedEncodingIgnored() {

‎build.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -444,7 +444,7 @@
444444
'Foundation/URLSession/NativeProtocol.swift',
445445
'Foundation/URLSession/TransferState.swift',
446446
'Foundation/URLSession/libcurl/libcurlHelpers.swift',
447-
'Foundation/URLSession/http/HTTPURLProtocol.swift',
447+
'Foundation/URLSession/http/HTTPURLProtocol.swift',
448448
'Foundation/UserDefaults.swift',
449449
'Foundation/NSUUID.swift',
450450
'Foundation/NSValue.swift',
@@ -501,6 +501,7 @@
501501
'TestFoundation/Resources/NSString-UTF16-LE-data.txt',
502502
'TestFoundation/Resources/NSString-UTF32-BE-data.txt',
503503
'TestFoundation/Resources/NSString-UTF32-LE-data.txt',
504+
'TestFoundation/Resources/NSString-ISO-8859-1-data.txt',
504505
'TestFoundation/Resources/NSXMLDocumentTestData.xml',
505506
'TestFoundation/Resources/PropertyList-1.0.dtd',
506507
'TestFoundation/Resources/NSXMLDTDTestData.xml',

0 commit comments

Comments
 (0)