Skip to content

[4.2] SR-7455: Allow NUL in Strings to match Darwin #1890

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 8, 2019
Merged

[4.2] SR-7455: Allow NUL in Strings to match Darwin #1890

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8412028
SR-7455: Allow NUL in Strings to match Darwin
spevans Aug 29, 2018
b935c5c
SR-7455: Minor speedups
spevans Aug 30, 2018
adb7935
SR-7455: Ask for ASCII when calling CFStringGetCStringPtr()
spevans Sep 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions DarwinCompatibilityTests.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
B917D32620B0DE2000728EE0 /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = B917D32520B0DE2000728EE0 /* main.swift */; };
B95788861F6FB9470003EB01 /* TestNSNumberBridging.swift in Sources */ = {isa = PBXBuildFile; fileRef = B95788851F6FB9470003EB01 /* TestNSNumberBridging.swift */; };
B9C89ED21F6BF67C00087AF4 /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = B9C89ED11F6BF67C00087AF4 /* XCTest.framework */; };
B987C65E2093C8AF0026B50D /* TestImports.swift in Sources */ = {isa = PBXBuildFile; fileRef = B987C65D2093C8AF0026B50D /* TestImports.swift */; };
B98E33E02136AC120044EBE9 /* TestFileWithZeros.txt in Resources */ = {isa = PBXBuildFile; fileRef = B98E33DF2136AC120044EBE9 /* TestFileWithZeros.txt */; };
B9C89F361F6BF89C00087AF4 /* TestScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9C89EE61F6BF88F00087AF4 /* TestScanner.swift */; };
B9C89F371F6BF89C00087AF4 /* TestNSValue.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9C89EE71F6BF88F00087AF4 /* TestNSValue.swift */; };
B9C89F381F6BF89C00087AF4 /* TestUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9C89EE81F6BF88F00087AF4 /* TestUtils.swift */; };
Expand Down Expand Up @@ -132,6 +134,8 @@
B917D32520B0DE2000728EE0 /* main.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = main.swift; path = TestFoundation/xdgTestHelper/main.swift; sourceTree = "<group>"; };
B95788851F6FB9470003EB01 /* TestNSNumberBridging.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = TestNSNumberBridging.swift; path = TestFoundation/TestNSNumberBridging.swift; sourceTree = "<group>"; };
B9C89EC11F6BF47D00087AF4 /* DarwinCompatibilityTests */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = DarwinCompatibilityTests; sourceTree = BUILT_PRODUCTS_DIR; };
B987C65D2093C8AF0026B50D /* TestImports.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = TestImports.swift; path = TestFoundation/TestImports.swift; sourceTree = "<group>"; };
B98E33DF2136AC120044EBE9 /* TestFileWithZeros.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; name = TestFileWithZeros.txt; path = TestFoundation/Resources/TestFileWithZeros.txt; sourceTree = "<group>"; };
B9C89ED11F6BF67C00087AF4 /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; };
B9C89ED71F6BF77E00087AF4 /* DarwinCompatibilityTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = DarwinCompatibilityTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
B9C89EDB1F6BF77E00087AF4 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
Expand Down Expand Up @@ -283,6 +287,10 @@
B9C89FA91F6DCAE700087AF4 /* NSXMLDTDTestData.xml */,
B9C89FB81F6DCAEB00087AF4 /* PropertyList-1.0.dtd */,
B9C89FB91F6DCAEB00087AF4 /* Test.plist */,
B98E33DF2136AC120044EBE9 /* TestFileWithZeros.txt */,
B917D32520B0DE2000728EE0 /* main.swift */,
B987C65D2093C8AF0026B50D /* TestImports.swift */,
B95788851F6FB9470003EB01 /* TestNSNumberBridging.swift */,
B9C89F891F6D4D9D00087AF4 /* HTTPServer.swift */,
B9C89F321F6BF89B00087AF4 /* TestAffineTransform.swift */,
B9C89EFB1F6BF89200087AF4 /* TestBundle.swift */,
Expand Down Expand Up @@ -486,6 +494,7 @@
B9C89FC11F6DCAEB00087AF4 /* Info.plist in Resources */,
B9C89FC21F6DCAEB00087AF4 /* NSKeyedUnarchiver-ArrayTest.plist in Resources */,
B9C89FC31F6DCAEB00087AF4 /* NSStringTestData.txt in Resources */,
B98E33E02136AC120044EBE9 /* TestFileWithZeros.txt in Resources */,
B9C89FC41F6DCAEB00087AF4 /* NSKeyedUnarchiver-OrderedSetTest.plist in Resources */,
B9C89FC51F6DCAEB00087AF4 /* NSString-UTF32-BE-data.txt in Resources */,
B9C89FC61F6DCAEB00087AF4 /* NSKeyedUnarchiver-URLTest.plist in Resources */,
Expand Down Expand Up @@ -515,6 +524,7 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
B987C65E2093C8AF0026B50D /* TestImports.swift in Sources */,
B95788861F6FB9470003EB01 /* TestNSNumberBridging.swift in Sources */,
B9C89F8B1F6D4DA900087AF4 /* HTTPServer.swift in Sources */,
B9C89F8C1F6D4DA900087AF4 /* TestHTTPCookieStorage.swift in Sources */,
Expand Down
4 changes: 4 additions & 0 deletions Foundation.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,7 @@
B933A79E1F3055F700FE6846 /* NSString-UTF32-BE-data.txt in Resources */ = {isa = PBXBuildFile; fileRef = B933A79C1F3055F600FE6846 /* NSString-UTF32-BE-data.txt */; };
B933A79F1F3055F700FE6846 /* NSString-UTF32-LE-data.txt in Resources */ = {isa = PBXBuildFile; fileRef = B933A79D1F3055F600FE6846 /* NSString-UTF32-LE-data.txt */; };
B951B5EC1F4E2A2000D8B332 /* TestNSLock.swift in Sources */ = {isa = PBXBuildFile; fileRef = B951B5EB1F4E2A2000D8B332 /* TestNSLock.swift */; };
B98E33DD2136AA740044EBE9 /* TestFileWithZeros.txt in Resources */ = {isa = PBXBuildFile; fileRef = B98E33DC2136AA740044EBE9 /* TestFileWithZeros.txt */; };
B9974B961EDF4A22007F15B8 /* TransferState.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9974B8F1EDF4A22007F15B8 /* TransferState.swift */; };
B9974B971EDF4A22007F15B8 /* MultiHandle.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9974B901EDF4A22007F15B8 /* MultiHandle.swift */; };
B9974B981EDF4A22007F15B8 /* libcurlHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9974B911EDF4A22007F15B8 /* libcurlHelpers.swift */; };
Expand Down Expand Up @@ -821,6 +822,7 @@
B933A79C1F3055F600FE6846 /* NSString-UTF32-BE-data.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = "NSString-UTF32-BE-data.txt"; sourceTree = "<group>"; };
B933A79D1F3055F600FE6846 /* NSString-UTF32-LE-data.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = "NSString-UTF32-LE-data.txt"; sourceTree = "<group>"; };
B951B5EB1F4E2A2000D8B332 /* TestNSLock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestNSLock.swift; sourceTree = "<group>"; };
B98E33DC2136AA740044EBE9 /* TestFileWithZeros.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = TestFileWithZeros.txt; sourceTree = "<group>"; };
B9974B8F1EDF4A22007F15B8 /* TransferState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TransferState.swift; sourceTree = "<group>"; };
B9974B901EDF4A22007F15B8 /* MultiHandle.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MultiHandle.swift; sourceTree = "<group>"; };
B9974B911EDF4A22007F15B8 /* libcurlHelpers.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = libcurlHelpers.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -1446,6 +1448,7 @@
EA66F6391BF1619600136161 /* Resources */ = {
isa = PBXGroup;
children = (
B98E33DC2136AA740044EBE9 /* TestFileWithZeros.txt */,
D370696D1C394FBF00295652 /* NSKeyedUnarchiver-RangeTest.plist */,
D3E8D6D41C36AC0C00295652 /* NSKeyedUnarchiver-RectTest.plist */,
D3E8D6D21C36982700295652 /* NSKeyedUnarchiver-EdgeInsetsTest.plist */,
Expand Down Expand Up @@ -2143,6 +2146,7 @@
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
B98E33DD2136AA740044EBE9 /* TestFileWithZeros.txt in Resources */,
B933A79E1F3055F700FE6846 /* NSString-UTF32-BE-data.txt in Resources */,
B933A79F1F3055F700FE6846 /* NSString-UTF32-LE-data.txt in Resources */,
D3A597F41C34142600295652 /* NSKeyedUnarchiver-NotificationTest.plist in Resources */,
Expand Down
23 changes: 16 additions & 7 deletions Foundation/String.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,22 +29,31 @@ extension String : _ObjectiveCBridgeable {
result = source._storage
} else if type(of: source) == _NSCFString.self {
let cf = unsafeBitCast(source, to: CFString.self)
if let str = CFStringGetCStringPtr(cf, CFStringEncoding(kCFStringEncodingUTF8)) {
result = String(cString: str)
let length = CFStringGetLength(cf)
if length == 0 {
result = ""
} else if let ptr = CFStringGetCStringPtr(cf, CFStringEncoding(kCFStringEncodingASCII)) {
// ASCII encoding has 1 byte per code point and CFStringGetLength() returned the length in
// codepoints so length should be the length of the ASCII string in bytes. We cant ask for the UTF-8
// encoding as some codepoints are multi-byte in UTF8 so the buffer length wouldn't be known.
// Note: CFStringGetCStringPtr(cf, CFStringEncoding(kCFStringEncodingUTF8)) does seems to return NULL
// for strings with multibyte UTF-8 but this isnt guaranteed or documented so ASCII is safer.
result = ptr.withMemoryRebound(to: UInt8.self, capacity: length) {
return String(decoding: UnsafeBufferPointer(start: $0, count: length), as: UTF8.self)
}
} else if let ptr = CFStringGetCharactersPtr(cf) {
result = String(decoding: UnsafeBufferPointer(start: ptr, count: length), as: UTF16.self)
} else {
let length = CFStringGetLength(cf)
let buffer = UnsafeMutablePointer<UniChar>.allocate(capacity: length)
CFStringGetCharacters(cf, CFRangeMake(0, length), buffer)

let str = String(decoding: UnsafeBufferPointer(start: buffer, count: length), as: UTF16.self)
result = String(decoding: UnsafeBufferPointer(start: buffer, count: length), as: UTF16.self)
buffer.deinitialize(count: length)
buffer.deallocate()
result = str
}
} else if type(of: source) == _NSCFConstantString.self {
let conststr = unsafeDowncast(source, to: _NSCFConstantString.self)
let str = String(decoding: UnsafeBufferPointer(start: conststr._ptr, count: Int(conststr._length)), as: UTF8.self)
result = str
result = String(decoding: UnsafeBufferPointer(start: conststr._ptr, count: Int(conststr._length)), as: UTF8.self)
} else {
let len = source.length
var characters = [unichar](repeating: 0, count: len)
Expand Down
Binary file added TestFoundation/Resources/TestFileWithZeros.txt
View file
Open in desktop
Binary file not shown.
4 changes: 2 additions & 2 deletions TestFoundation/TestJSONSerialization.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1367,7 +1367,7 @@ extension TestJSONSerialization {
let result = try JSONSerialization.writeJSONObject(dict, toStream: outputStream, options: [])
outputStream.close()
if(result > -1) {
XCTAssertEqual(NSString(bytes: buffer, length: buffer.count, encoding: String.Encoding.utf8.rawValue), "{\"a\":{\"b\":1}}")
XCTAssertEqual(NSString(bytes: buffer, length: buffer.firstIndex(of: 0) ?? buffer.count, encoding: String.Encoding.utf8.rawValue), "{\"a\":{\"b\":1}}")
}
} catch {
XCTFail("Error thrown: \(error)")
Expand All @@ -1390,7 +1390,7 @@ extension TestJSONSerialization {
let resultRead: Int = fileStream.read(&buffer, maxLength: buffer.count)
fileStream.close()
if(resultRead > -1){
XCTAssertEqual(NSString(bytes: buffer, length: buffer.count, encoding: String.Encoding.utf8.rawValue), "{\"a\":{\"b\":1}}")
XCTAssertEqual(NSString(bytes: buffer, length: buffer.firstIndex(of: 0) ?? buffer.count, encoding: String.Encoding.utf8.rawValue), "{\"a\":{\"b\":1}}")
}
}
removeTestFile(filePath)
Expand Down
9 changes: 5 additions & 4 deletions TestFoundation/TestNSData.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1466,11 +1466,12 @@ extension TestNSData {
if let contents = contents {
XCTAssertTrue(contents.length > 0)
let ptr = UnsafeMutableRawPointer(mutating: contents.bytes)
let str = String(bytesNoCopy: ptr, length: contents.length,
encoding: .ascii, freeWhenDone: false)
XCTAssertNotNil(str)
if let str = str {
var zeroIdx = contents.range(of: Data([0]), in: NSMakeRange(0, contents.length)).location
if zeroIdx == NSNotFound { zeroIdx = contents.length }
if let str = String(bytesNoCopy: ptr, length: zeroIdx, encoding: .ascii, freeWhenDone: false) {
XCTAssertTrue(str.hasSuffix("TestFoundation"))
} else {
XCTFail("Cant create String")
}
}

Expand Down
75 changes: 75 additions & 0 deletions TestFoundation/TestNSString.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ class TestNSString: LoopbackServerTest {

static var allTests: [(String, (TestNSString) -> () throws -> Void)] {
return [
("test_initData", test_initData),
("test_boolValue", test_boolValue ),
("test_BridgeConstruction", test_BridgeConstruction ),
("test_integerValue", test_integerValue ),
Expand Down Expand Up @@ -95,6 +96,70 @@ class TestNSString: LoopbackServerTest {
]
}

func test_initData() {
let testString = "\u{00} This is a test string"
let data = testString.data(using: .utf8)!
XCTAssertEqual(data.count, 23)
_ = data.withUnsafeBytes { (bytes: UnsafePointer<UInt8>) in
if let text1 = NSString(bytes: bytes , length: data.count, encoding: String.Encoding.utf8.rawValue) {
XCTAssertEqual(text1.length, data.count)
XCTAssertEqual(text1, testString as NSString)
} else {
XCTFail("Cant convert Data to NSString")
}
}

if let text2 = String(data: data, encoding: .utf8) {
XCTAssertEqual(text2.count, data.count)
XCTAssertEqual(text2, testString)
} else {
XCTFail("Cant convert Data to String")
}

// Test multibyte UTF8 and UTF16
// kra ("ĸ") has codepoint value 312,
// as UTF-8 bytes it is 0xC4 0xB8
// as UTF-16 bytes it is 0x1, 0x38
let kra = "ĸ"
let utf8KraData = Data(bytes: [0xc4, 0xb8])
if let utf8kra = utf8KraData.withUnsafeBytes( { (bytes: UnsafePointer<UInt8>) in
return NSString(bytes: bytes, length: utf8KraData.count, encoding: String.Encoding.utf8.rawValue)
}) {
XCTAssertEqual(kra.count, 1)
XCTAssertEqual(kra.utf8.count, 2)
XCTAssertEqual(kra.utf16.count, 1)
XCTAssertEqual(kra, utf8kra as String)
} else {
XCTFail("Cant create UTF8 kra")
}

let utf16KraData = Data(bytes: [0x1, 0x38])
if let utf16kra = utf16KraData.withUnsafeBytes( { (bytes: UnsafePointer<UInt8>) in
return NSString(bytes: bytes, length: utf16KraData.count, encoding: String.Encoding.utf16.rawValue)
}) {
XCTAssertEqual(kra.count, 1)
XCTAssertEqual(kra.utf8.count, 2)
XCTAssertEqual(kra.utf16.count, 1)
XCTAssertEqual(kra, utf16kra as String)
} else {
XCTFail("Cant create UTF16 kra")
}

// Test a large string > 255 characters
let largeString = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut at tincidunt arcu. Suspendisse nec sodales erat, sit amet imperdiet ipsum. Etiam sed ornare felis. Nunc mauris turpis, bibendum non lectus quis, malesuada placerat turpis. Nam adipiscing non massa et semper. Nulla convallis semper bibendum."
XCTAssertTrue(largeString.count > 255)
let largeData = largeString.data(using: .utf8)!
if let largeText = largeData.withUnsafeBytes( { (bytes: UnsafePointer<UInt8>) in
return NSString(bytes: bytes, length: largeData.count, encoding: String.Encoding.ascii.rawValue)
}) {
XCTAssertEqual(largeText.length, largeString.count)
XCTAssertEqual(largeText.length, largeData.count)
XCTAssertEqual(largeString, largeText as String)
} else {
XCTFail("Cant convert large Data string to String")
}
}

func test_boolValue() {
let trueStrings: [NSString] = ["t", "true", "TRUE", "tRuE", "yes", "YES", "1", "+000009"]
for string in trueStrings {
Expand Down Expand Up @@ -303,6 +368,16 @@ class TestNSString: LoopbackServerTest {
if let contents = contents {
XCTAssertEqual(contents, "This file is encoded as ISO-8859-1\nÀÁÂÃÄÅÿ\n±\n")
}

guard let zeroFileURL = testBundle().url(forResource: "TestFileWithZeros", withExtension: "txt") else {
XCTFail("Cant get URL for TestFileWithZeros.txt")
return
}
guard let zeroString = try? String(contentsOf: zeroFileURL, encoding: .utf8) else {
XCTFail("Cant create string from \(zeroFileURL)")
return
}
XCTAssertEqual(zeroString, "Some\u{00}text\u{00}with\u{00}NUL\u{00}bytes\u{00}instead\u{00}of\u{00}spaces.\u{00}\n")
}

func test_FromContentOfFileUsedEncodingIgnored() {
Expand Down
10 changes: 5 additions & 5 deletions TestFoundation/TestStream.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ class TestStream : XCTestCase {
let result: Int = dataStream.read(&buffer, maxLength: buffer.count)
dataStream.close()
XCTAssertEqual(Stream.Status.closed, dataStream.streamStatus)
if(result > 0){
let output = NSString(bytes: &buffer, length: buffer.count, encoding: String.Encoding.utf8.rawValue)
if(result > 0) {
let output = NSString(bytes: &buffer, length: buffer.firstIndex(of: 0) ?? buffer.count, encoding: String.Encoding.utf8.rawValue)
XCTAssertEqual(message, output!)
}
}
Expand All @@ -64,7 +64,7 @@ class TestStream : XCTestCase {
XCTAssertEqual(Stream.Status.closed, urlStream.streamStatus)
XCTAssertEqual(messageData.count, result)
if(result > 0) {
let output = NSString(bytes: &buffer, length: buffer.count, encoding: String.Encoding.utf8.rawValue)
let output = NSString(bytes: &buffer, length: buffer.firstIndex(of: 0) ?? buffer.count, encoding: String.Encoding.utf8.rawValue)
XCTAssertEqual(message, output!)
}
}
Expand All @@ -90,8 +90,8 @@ class TestStream : XCTestCase {
fileStream.close()
XCTAssertEqual(Stream.Status.closed, fileStream.streamStatus)
XCTAssertEqual(messageData.count, result)
if(result > 0){
let output = NSString(bytes: &buffer, length: buffer.count, encoding: String.Encoding.utf8.rawValue)
if(result > 0) {
let output = NSString(bytes: &buffer, length: buffer.firstIndex(of: 0) ?? buffer.count, encoding: String.Encoding.utf8.rawValue)
XCTAssertEqual(message, output!)
}
}
Expand Down
1 change: 1 addition & 0 deletions build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,7 @@
'TestFoundation/Resources/NSKeyedUnarchiver-URLTest.plist',
'TestFoundation/Resources/NSKeyedUnarchiver-UUIDTest.plist',
'TestFoundation/Resources/NSKeyedUnarchiver-OrderedSetTest.plist',
'TestFoundation/Resources/TestFileWithZeros.txt',
])

# TODO: Probably this should be another 'product', but for now it's simply a phase
Expand Down