Cherry-picking some URLSession fixes on to 3.1

Foundation/NSURLSession/EasyHandle.swift

@@ -56,6 +56,8 @@ internal final class _EasyHandle {
     fileprivate var headerList: _CurlStringList?
     fileprivate var pauseState: _PauseState = []
     internal var fileLength: Int64 = 0
+    internal var timeoutTimer: _TimeoutSource!
+
     init(delegate: _EasyHandleDelegate) {
         self.delegate = delegate
         setupCallbacks()
@@ -394,31 +396,47 @@ fileprivate extension _EasyHandle {
 }
 
 fileprivate extension _EasyHandle {
+
+    func resetTimer() {
+        //simply create a new timer with the same queue, timeout and handler
+        //this must cancel the old handler and reset the timer
+        timeoutTimer = _TimeoutSource(queue: timeoutTimer.queue, milliseconds: timeoutTimer.milliseconds, handler: timeoutTimer.handler)
+    }
+
     /// Forward the libcurl callbacks into Swift methods
     func setupCallbacks() {
         // write
         try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionWRITEDATA, UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())).asError()
 
         try! CFURLSession_easy_setopt_wc(rawHandle, CFURLSessionOptionWRITEFUNCTION) { (data: UnsafeMutablePointer<Int8>, size: Int, nmemb: Int, userdata: UnsafeMutableRawPointer?) -> Int in
             guard let handle = _EasyHandle.from(callbackUserData: userdata) else { return 0 }
+            defer {
+                handle.resetTimer()
+            }
             return handle.didReceive(data: data, size: size, nmemb: nmemb)
-            }.asError()
+        }.asError()
 
         // read
         try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionREADDATA, UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())).asError()
         try! CFURLSession_easy_setopt_wc(rawHandle, CFURLSessionOptionREADFUNCTION) { (data: UnsafeMutablePointer<Int8>, size: Int, nmemb: Int, userdata: UnsafeMutableRawPointer?) -> Int in
             guard let handle = _EasyHandle.from(callbackUserData: userdata) else { return 0 }
+            defer {
+                handle.resetTimer()
+            }
             return handle.fill(writeBuffer: data, size: size, nmemb: nmemb)
-            }.asError()
+        }.asError()
 
         // header
         try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionHEADERDATA, UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())).asError()
         try! CFURLSession_easy_setopt_wc(rawHandle, CFURLSessionOptionHEADERFUNCTION) { (data: UnsafeMutablePointer<Int8>, size: Int, nmemb: Int, userdata: UnsafeMutableRawPointer?) -> Int in
             guard let handle = _EasyHandle.from(callbackUserData: userdata) else { return 0 }
+            defer {
+                handle.resetTimer()
+            }
             var length = Double()
             try! CFURLSession_easy_getinfo_double(handle.rawHandle, CFURLSessionInfoCONTENT_LENGTH_DOWNLOAD, &length).asError()
             return handle.didReceive(headerData: data, size: size, nmemb: nmemb, fileLength: length)
-            }.asError()
+        }.asError()
 
         // socket options
         try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionSOCKOPTDATA, UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())).asError()
@@ -431,7 +449,7 @@ fileprivate extension _EasyHandle {
             } catch {
                 return 1
             }
-            }.asError()
+        }.asError()
         // seeking in input stream
         try! CFURLSession_easy_setopt_ptr(rawHandle, CFURLSessionOptionSEEKDATA, UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())).asError()
         try! CFURLSession_easy_setopt_seek(rawHandle, CFURLSessionOptionSEEKFUNCTION, { (userdata, offset, origin) -> Int32 in

Foundation/NSURLSession/MultiHandle.swift

@@ -302,11 +302,15 @@ fileprivate extension URLSession._MultiHandle._SocketRegisterAction {
 
 /// A helper class that wraps a libdispatch timer.
 ///
-/// Used to implement the timeout of `URLSession.MultiHandle`.
-fileprivate class _TimeoutSource {
+/// Used to implement the timeout of `URLSession.MultiHandle` and `URLSession.EasyHandle`
+class _TimeoutSource {
     let rawSource: DispatchSource 
     let milliseconds: Int
+    let queue: DispatchQueue        //needed to restart the timer for EasyHandles
+    let handler: DispatchWorkItem   //needed to restart the timer for EasyHandles
     init(queue: DispatchQueue, milliseconds: Int, handler: DispatchWorkItem) {
+        self.queue = queue
+        self.handler = handler
         self.milliseconds = milliseconds
         self.rawSource = DispatchSource.makeTimerSource(queue: queue) as! DispatchSource
 

Foundation/NSURLSession/NSURLSessionTask.swift

@@ -548,7 +548,38 @@ fileprivate extension URLSessionTask {
 
         // HTTP Options:
         easyHandle.set(followLocation: false)
-        easyHandle.set(customHeaders: curlHeaders(for: request))
+
+        // The httpAdditionalHeaders from session configuration has to be added to the request.
+        // The request.allHTTPHeaders can override the httpAdditionalHeaders elements. Add the
+        // httpAdditionalHeaders from session configuration first and then append/update the
+        // request.allHTTPHeaders so that request.allHTTPHeaders can override httpAdditionalHeaders.
+
+        let httpSession = session as! URLSession
+        var httpHeaders: [AnyHashable : Any]?
+
+        if let hh = httpSession.configuration.httpAdditionalHeaders {
+            httpHeaders = hh
+        }
+
+        if let hh = currentRequest?.allHTTPHeaderFields {
+            if httpHeaders == nil {
+                httpHeaders = hh
+            } else {
+                hh.forEach {
+                    httpHeaders![$0] = $1
+                }
+            }
+        }
+
+        let customHeaders: [String]
+        let headersForRequest = curlHeaders(for: httpHeaders)
+        if ((request.httpMethod == "POST") && (request.value(forHTTPHeaderField: "Content-Type") == nil)) {
+            customHeaders = headersForRequest + ["Content-Type:application/x-www-form-urlencoded"]
+        } else {
+            customHeaders = headersForRequest
+        }
+
+        easyHandle.set(customHeaders: customHeaders)
 
         //Options unavailable on Ubuntu 14.04 (libcurl 7.36)
         //TODO: Introduce something like an #if
@@ -557,15 +588,19 @@ fileprivate extension URLSessionTask {
 
         //set the request timeout
         //TODO: the timeout value needs to be reset on every data transfer
-        let s = session as! URLSession
-        easyHandle.set(timeout: Int(s.configuration.timeoutIntervalForRequest))
+        let timeoutInterval = Int(httpSession.configuration.timeoutIntervalForRequest) * 1000
+        let timeoutHandler = DispatchWorkItem { [weak self] in
+            guard let currentTask = self else { fatalError("Timeout on a task that doesn't exist") } //this guard must always pass
+            currentTask.internalState = .transferFailed
+            let urlError = URLError(_nsError: NSError(domain: NSURLErrorDomain, code: NSURLErrorTimedOut, userInfo: nil))
+            currentTask.completeTask(withError: urlError)
+        }
+        easyHandle.timeoutTimer = _TimeoutSource(queue: workQueue, milliseconds: timeoutInterval, handler: timeoutHandler)
 
         easyHandle.set(automaticBodyDecompression: true)
         easyHandle.set(requestMethod: request.httpMethod ?? "GET")
         if request.httpMethod == "HEAD" {
             easyHandle.set(noBody: true)
-        } else if ((request.httpMethod == "POST") && (request.value(forHTTPHeaderField: "Content-Type") == nil)) {
-            easyHandle.set(customHeaders: ["Content-Type:application/x-www-form-urlencoded"])
         }
     }
 }
@@ -579,10 +614,11 @@ fileprivate extension URLSessionTask {
     /// expects.
     ///
     /// - SeeAlso: https://curl.haxx.se/libcurl/c/CURLOPT_HTTPHEADER.html
-    func curlHeaders(for request: URLRequest) -> [String] {
+    func curlHeaders(for httpHeaders: [AnyHashable : Any]?) -> [String] {
         var result: [String] = []
         var names = Set<String>()
-        if let hh = currentRequest?.allHTTPHeaderFields {
+        if httpHeaders != nil {
+            let hh = httpHeaders as! [String:String]
             hh.forEach {
                 let name = $0.0.lowercased()
                 guard !names.contains(name) else { return }
@@ -819,6 +855,9 @@ extension URLSessionTask {
         }
         self.response = response
 
+        //We don't want a timeout to be triggered after this. The timeout timer needs to be cancelled.
+        easyHandle.timeoutTimer = nil
+
         //because we deregister the task with the session on internalState being set to taskCompleted
         //we need to do the latter after the delegate/handler was notified/invoked
         switch session.behaviour(for: self) {
@@ -870,6 +909,10 @@ extension URLSessionTask {
         guard case .transferFailed = internalState else {
             fatalError("Trying to complete the task, but its transfer isn't complete / failed.")
         }
+
+        //We don't want a timeout to be triggered after this. The timeout timer needs to be cancelled.
+        easyHandle.timeoutTimer = nil
+
         switch session.behaviour(for: self) {
         case .taskDelegate(let delegate):
             guard let s = session as? URLSession else { fatalError() }

TestFoundation/HTTPServer.swift

@@ -77,6 +77,7 @@ class _TCPSocket {
             return sockaddr_in(sin_len: 0, sin_family: sa_family_t(AF_INET), sin_port: CFSwapInt16HostToBig(port), sin_addr: in_addr(s_addr: INADDR_ANY), sin_zero: (0,0,0,0,0,0,0,0) )
         #endif
     }
+
     func acceptConnection(notify: ServerSemaphore) throws {
         _ = try attempt("listen", valid: isZero, listen(listenSocket, SOMAXCONN))
         try socketAddress.withMemoryRebound(to: sockaddr.self, capacity: MemoryLayout<sockaddr>.size, {
@@ -92,10 +93,32 @@ class _TCPSocket {
         _ = try attempt("read", valid: isNotNegative, CInt(read(connectionSocket, &buffer, 4096)))
         return String(cString: &buffer)
     }
+
+    func split(_ str: String, _ count: Int) -> [String] {
+        return stride(from: 0, to: str.characters.count, by: count).map { i -> String in
+            let startIndex = str.index(str.startIndex, offsetBy: i)
+            let endIndex   = str.index(startIndex, offsetBy: count, limitedBy: str.endIndex) ?? str.endIndex
+            return str[startIndex..<endIndex]
+        }
+    }
 
-    func writeData(data: String) throws {
-        var bytes = Array(data.utf8)
-        _  = try attempt("write", valid: isNotNegative, CInt(write(connectionSocket, &bytes, data.utf8.count))) 
+    func writeData(header: String, body: String, sendDelay: TimeInterval? = nil, bodyChunks: Int? = nil) throws {
+        var header = Array(header.utf8)
+        _  = try attempt("write", valid: isNotNegative, CInt(write(connectionSocket, &header, header.count)))
+
+        if let sendDelay = sendDelay, let bodyChunks = bodyChunks {
+            let count = max(1, Int(Double(body.utf8.count) / Double(bodyChunks)))
+            let texts = split(body, count)
+
+            for item in texts {
+                sleep(UInt32(sendDelay))
+                var bytes = Array(item.utf8)
+                _  = try attempt("write", valid: isNotNegative, CInt(write(connectionSocket, &bytes, bytes.count)))
+            }
+        } else {
+            var bytes = Array(body.utf8)
+            _  = try attempt("write", valid: isNotNegative, CInt(write(connectionSocket, &bytes, bytes.count)))
+        }
     }
 
     func shutdown() {
@@ -128,8 +151,24 @@ class _HTTPServer {
        return _HTTPRequest(request: try socket.readData()) 
     }
 
-    public func respond(with response: _HTTPResponse) throws {
-        try socket.writeData(data: response.description)
+    public func respond(with response: _HTTPResponse, startDelay: TimeInterval? = nil, sendDelay: TimeInterval? = nil, bodyChunks: Int? = nil) throws {
+        let semaphore = DispatchSemaphore(value: 0)
+        let deadlineTime: DispatchTime
+
+        if let startDelay = startDelay {
+           deadlineTime = .now() + .seconds(Int(startDelay))
+        } else {
+            deadlineTime = .now()
+        }
+
+        DispatchQueue.main.asyncAfter(deadline: deadlineTime) {
+            do {
+                try self.socket.writeData(header: response.header, body: response.body, sendDelay: sendDelay, bodyChunks: bodyChunks)
+                semaphore.signal()
+            } catch { }
+        }
+        semaphore.wait()
+
     } 
 }
 
@@ -152,6 +191,14 @@ struct _HTTPRequest {
         body = lines.last!
     }
 
+    public func getCommaSeparatedHeaders() -> String {
+        var allHeaders = ""
+        for header in headers {
+            allHeaders += header + ","
+        }
+        return allHeaders
+    }
+
 }
 
 struct _HTTPResponse {
@@ -160,17 +207,17 @@ struct _HTTPResponse {
     }
     private let responseCode: Response
     private let headers: String
-    private let body: String
+    public let body: String
 
     public init(response: Response, headers: String = _HTTPUtils.EMPTY, body: String) {
         self.responseCode = response
         self.headers = headers
         self.body = body
     }
 
-    public var description: String {
+    public var header: String {
         let statusLine = _HTTPUtils.VERSION + _HTTPUtils.SPACE + "\(responseCode.rawValue)" + _HTTPUtils.SPACE + "\(responseCode)"
-        return statusLine + (headers != _HTTPUtils.EMPTY ? _HTTPUtils.CRLF + headers : _HTTPUtils.EMPTY) + _HTTPUtils.CRLF2 + body
+        return statusLine + (headers != _HTTPUtils.EMPTY ? _HTTPUtils.CRLF + headers : _HTTPUtils.EMPTY) + _HTTPUtils.CRLF2
     }
 }
 
@@ -181,32 +228,44 @@ public class TestURLSessionServer {
                                      "USA":"Washington, D.C.",
                                      "country.txt": "A country is a region that is identified as a distinct national entity in political geography"]
     let httpServer: _HTTPServer
+    let startDelay: TimeInterval?
+    let sendDelay: TimeInterval?
+    let bodyChunks: Int?
 
-    public init (port: UInt16) throws {
+    public init (port: UInt16, startDelay: TimeInterval? = nil, sendDelay: TimeInterval? = nil, bodyChunks: Int? = nil) throws {
         httpServer = try _HTTPServer.create(port: port)
+        self.startDelay = startDelay
+        self.sendDelay = sendDelay
+        self.bodyChunks = bodyChunks
     }
     public func start(started: ServerSemaphore) throws {
         started.signal()
         try httpServer.listen(notify: started)
     }
 
     public func readAndRespond() throws {
-        try httpServer.respond(with: process(request: httpServer.request()))
-    } 
+        try httpServer.respond(with: process(request: httpServer.request()), startDelay: self.startDelay, sendDelay: self.sendDelay, bodyChunks: self.bodyChunks)
+    }
 
     func process(request: _HTTPRequest) -> _HTTPResponse {
-        if request.method == .GET {
-            return getResponse(uri: request.uri)
+        if request.method == .GET || request.method == .POST {
+            return getResponse(request: request)
         } else {
             fatalError("Unsupported method!")
         }
     }
 
-    func getResponse(uri: String) -> _HTTPResponse {
+    func getResponse(request: _HTTPRequest) -> _HTTPResponse {
+        let uri = request.uri
         if uri == "/country.txt" {
             let text = capitals[String(uri.characters.dropFirst())]!
             return _HTTPResponse(response: .OK, headers: "Content-Length: \(text.characters.count)", body: text)
         }
+
+        if uri == "/requestHeaders" {
+            let text = request.getCommaSeparatedHeaders()
+            return _HTTPResponse(response: .OK, headers: "Content-Length: \(text.characters.count)", body: text)
+        }
         return _HTTPResponse(response: .OK, body: capitals[String(uri.characters.dropFirst())]!) 
     }