Merge pull request #496 from justcoding121/master

justcoding121 · web-flow · commit 08167b4d678d · 2018-09-27T19:59:28.000-04:00
Dns resolve from code
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -15,7 +15,7 @@
         "**/*.Basic.csproj/": true,
         "**/*.Docs.csproj/": true,
         "**/*.Proxy.csproj/": true,
-        "**/*.Proxy.csproj" : true
+        "**/*.Mono.csproj" : true
     },
     "search.exclude": {
         "**/.build": true,
@@ -32,6 +32,6 @@
         "**/*.Basic.csproj/": true,
         "**/*.Docs.csproj/": true,
         "**/*.Proxy.csproj/": true,
-        "**/*.Proxy.csproj" : true
+        "**/*.Mono.csproj" : true
     }
 }
diff --git a/omnisharp.json b/omnisharp.json
@@ -1,6 +1,7 @@
 {
     "fileOptions": {
       "excludeSearchPatterns": [
+        "**/*.sln",
         "**/*.Docs.csproj",
         "**/tests/",
         "**/Titanium.Web.Proxy.Examples.Wpf/",
diff --git a/src/Titanium.Web.Proxy/Network/Tcp/TcpConnectionFactory.cs b/src/Titanium.Web.Proxy/Network/Tcp/TcpConnectionFactory.cs
@@ -22,6 +22,7 @@ namespace Titanium.Web.Proxy.Network.Tcp
     /// </summary>
     internal class TcpConnectionFactory : IDisposable
     {
+
         //Tcp server connection pool cache
         private readonly ConcurrentDictionary<string, ConcurrentQueue<TcpServerConnection>> cache
             = new ConcurrentDictionary<string, ConcurrentQueue<TcpServerConnection>>();
@@ -74,10 +75,10 @@ internal string GetConnectionCacheKey(string remoteHostName, int remotePort,
         /// <summary>
         ///     Gets the connection cache key.
         /// </summary>
-        /// <param name="args">The session event arguments.</param>
+        /// <param name="session">The session event arguments.</param>
         /// <param name="applicationProtocol"></param>
         /// <returns></returns>
-        internal async Task<string> GetConnectionCacheKey(ProxyServer server, SessionEventArgsBase args,
+        internal async Task<string> GetConnectionCacheKey(ProxyServer server, SessionEventArgsBase session,
             SslApplicationProtocol applicationProtocol)
         {
             List<SslApplicationProtocol> applicationProtocols = null;
@@ -88,32 +89,32 @@ internal async Task<string> GetConnectionCacheKey(ProxyServer server, SessionEve
 
             ExternalProxy customUpStreamProxy = null;
 
-            bool isHttps = args.IsHttps;
+            bool isHttps = session.IsHttps;
             if (server.GetCustomUpStreamProxyFunc != null)
             {
-                customUpStreamProxy = await server.GetCustomUpStreamProxyFunc(args);
+                customUpStreamProxy = await server.GetCustomUpStreamProxyFunc(session);
             }
 
-            args.CustomUpStreamProxyUsed = customUpStreamProxy;
+            session.CustomUpStreamProxyUsed = customUpStreamProxy;
 
             return GetConnectionCacheKey(
-                args.WebSession.Request.RequestUri.Host,
-                args.WebSession.Request.RequestUri.Port,
+                session.WebSession.Request.RequestUri.Host,
+                session.WebSession.Request.RequestUri.Port,
                 isHttps, applicationProtocols,
-                server, args.WebSession.UpStreamEndPoint ?? server.UpStreamEndPoint,
+                server, session.WebSession.UpStreamEndPoint ?? server.UpStreamEndPoint,
                 customUpStreamProxy ?? (isHttps ? server.UpStreamHttpsProxy : server.UpStreamHttpProxy));
         }
 
 
         /// <summary>
         ///     Create a server connection.
         /// </summary>
-        /// <param name="args">The session event arguments.</param>
+        /// <param name="session">The session event arguments.</param>
         /// <param name="isConnect">Is this a CONNECT request.</param>
         /// <param name="applicationProtocol"></param>
         /// <param name="cancellationToken">The cancellation token for this async task.</param>
         /// <returns></returns>
-        internal Task<TcpServerConnection> GetServerConnection(ProxyServer server, SessionEventArgsBase args, bool isConnect,
+        internal Task<TcpServerConnection> GetServerConnection(ProxyServer server, SessionEventArgsBase session, bool isConnect,
             SslApplicationProtocol applicationProtocol, bool noCache, CancellationToken cancellationToken)
         {
             List<SslApplicationProtocol> applicationProtocols = null;
@@ -122,36 +123,36 @@ internal Task<TcpServerConnection> GetServerConnection(ProxyServer server, Sessi
                 applicationProtocols = new List<SslApplicationProtocol> { applicationProtocol };
             }
 
-            return GetServerConnection(server, args, isConnect, applicationProtocols, noCache, cancellationToken);
+            return GetServerConnection(server, session, isConnect, applicationProtocols, noCache, cancellationToken);
         }
 
         /// <summary>
         ///     Create a server connection.
         /// </summary>
-        /// <param name="args">The session event arguments.</param>
+        /// <param name="session">The session event arguments.</param>
         /// <param name="isConnect">Is this a CONNECT request.</param>
         /// <param name="applicationProtocols"></param>
         /// <param name="cancellationToken">The cancellation token for this async task.</param>
         /// <returns></returns>
-        internal async Task<TcpServerConnection> GetServerConnection(ProxyServer server, SessionEventArgsBase args, bool isConnect,
+        internal async Task<TcpServerConnection> GetServerConnection(ProxyServer server, SessionEventArgsBase session, bool isConnect,
             List<SslApplicationProtocol> applicationProtocols, bool noCache, CancellationToken cancellationToken)
         {
             ExternalProxy customUpStreamProxy = null;
 
-            bool isHttps = args.IsHttps;
+            bool isHttps = session.IsHttps;
             if (server.GetCustomUpStreamProxyFunc != null)
             {
-                customUpStreamProxy = await server.GetCustomUpStreamProxyFunc(args);
+                customUpStreamProxy = await server.GetCustomUpStreamProxyFunc(session);
             }
 
-            args.CustomUpStreamProxyUsed = customUpStreamProxy;
+            session.CustomUpStreamProxyUsed = customUpStreamProxy;
 
             return await GetServerConnection(
-                args.WebSession.Request.RequestUri.Host,
-                args.WebSession.Request.RequestUri.Port,
-                args.WebSession.Request.HttpVersion,
+                session.WebSession.Request.RequestUri.Host,
+                session.WebSession.Request.RequestUri.Port,
+                session.WebSession.Request.HttpVersion,
                 isHttps, applicationProtocols, isConnect,
-                server, args.WebSession.UpStreamEndPoint ?? server.UpStreamEndPoint,
+                server, session, session.WebSession.UpStreamEndPoint ?? server.UpStreamEndPoint,
                 customUpStreamProxy ?? (isHttps ? server.UpStreamHttpsProxy : server.UpStreamHttpProxy),
                 noCache, cancellationToken);
         }
@@ -172,7 +173,7 @@ internal async Task<TcpServerConnection> GetServerConnection(ProxyServer server,
         /// <returns></returns>
         internal async Task<TcpServerConnection> GetServerConnection(string remoteHostName, int remotePort,
             Version httpVersion, bool isHttps, List<SslApplicationProtocol> applicationProtocols, bool isConnect,
-            ProxyServer proxyServer, IPEndPoint upStreamEndPoint, ExternalProxy externalProxy,
+            ProxyServer proxyServer, SessionEventArgsBase session, IPEndPoint upStreamEndPoint, ExternalProxy externalProxy,
             bool noCache, CancellationToken cancellationToken)
         {
             var cacheKey = GetConnectionCacheKey(remoteHostName, remotePort,
@@ -203,7 +204,7 @@ internal async Task<TcpServerConnection> GetServerConnection(string remoteHostNa
             }
 
             var connection = await createServerConnection(remoteHostName, remotePort, httpVersion, isHttps,
-                applicationProtocols, isConnect, proxyServer, upStreamEndPoint, externalProxy, cancellationToken);
+                applicationProtocols, isConnect, proxyServer, session, upStreamEndPoint, externalProxy, cancellationToken);
 
             connection.CacheKey = cacheKey;
 
@@ -220,13 +221,14 @@ internal async Task<TcpServerConnection> GetServerConnection(string remoteHostNa
         /// <param name="applicationProtocols">The list of HTTPS application level protocol to negotiate if needed.</param>
         /// <param name="isConnect">Is this a CONNECT request.</param>
         /// <param name="proxyServer">The current ProxyServer instance.</param>
+        /// <param name="session">The http session.</param>
         /// <param name="upStreamEndPoint">The local upstream endpoint to make request via.</param>
         /// <param name="externalProxy">The external proxy to make request via.</param>
         /// <param name="cancellationToken">The cancellation token for this async task.</param>
         /// <returns></returns>
         private async Task<TcpServerConnection> createServerConnection(string remoteHostName, int remotePort,
             Version httpVersion, bool isHttps, List<SslApplicationProtocol> applicationProtocols, bool isConnect,
-            ProxyServer proxyServer, IPEndPoint upStreamEndPoint, ExternalProxy externalProxy,
+            ProxyServer proxyServer, SessionEventArgsBase session, IPEndPoint upStreamEndPoint, ExternalProxy externalProxy,
             CancellationToken cancellationToken)
         {
             //deny connection to proxy end points to avoid infinite connection loop.
@@ -282,16 +284,37 @@ private async Task<TcpServerConnection> createServerConnection(string remoteHost
                     tcpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
                 }
 
-                // If this proxy uses another external proxy then create a tunnel request for HTTP/HTTPS connections
-                if (useUpstreamProxy)
+                var hostname = useUpstreamProxy ? externalProxy.HostName : remoteHostName;
+                var port = useUpstreamProxy ? externalProxy.Port : remotePort;
+
+                var ipHostEntry = await Dns.GetHostEntryAsync(hostname);
+                if (ipHostEntry == null || ipHostEntry.AddressList.Length == 0)
                 {
-                    await tcpClient.ConnectAsync(externalProxy.HostName, externalProxy.Port);
+                    throw new Exception($"Could not resolve the hostname {hostname}");
                 }
-                else
+
+                session.TimeLine["Dns Resolved"] = DateTime.Now;
+
+                var ipAddresses = ipHostEntry.AddressList;
+
+                for (int i = 0; i < ipAddresses.Length; i++)
                 {
-                    await tcpClient.ConnectAsync(remoteHostName, remotePort);
+                    try
+                    {
+                        await tcpClient.ConnectAsync(ipAddresses[i], port);
+                        break;
+                    }
+                    catch (Exception e)
+                    {
+                        if (i == ipAddresses.Length - 1)
+                        {
+                            throw new Exception($"Could not establish connection to {hostname}", e);
+                        }
+                    }
                 }
 
+                session.TimeLine["Connection Established"] = DateTime.Now;
+
                 await proxyServer.InvokeConnectionCreateEvent(tcpClient, false);
 
                 stream = new CustomBufferedStream(tcpClient.GetStream(), proxyServer.BufferPool, proxyServer.BufferSize);
@@ -347,6 +370,7 @@ private async Task<TcpServerConnection> createServerConnection(string remoteHost
 #if NETCOREAPP2_1
                     negotiatedApplicationProtocol = sslStream.NegotiatedApplicationProtocol;
 #endif
+                    session.TimeLine["HTTPS Established"] = DateTime.Now;
                 }
             }
             catch (Exception)
diff --git a/src/Titanium.Web.Proxy/ProxyServer.cs b/src/Titanium.Web.Proxy/ProxyServer.cs
@@ -94,8 +94,6 @@ public ProxyServer(string rootCertificateName, string rootCertificateIssuerName,
             bool userTrustRootCertificate = true, bool machineTrustRootCertificate = false,
             bool trustRootCertificateAsAdmin = false)
         {
-            // default values
-            ConnectionTimeOutSeconds = 60;
 
             if (BufferPool == null)
             {
@@ -186,7 +184,7 @@ public ProxyServer(string rootCertificateName, string rootCertificateIssuerName,
         ///     This will also determine the pool eviction time when connection pool is enabled.
         ///     Default value is 60 seconds.
         /// </summary>
-        public int ConnectionTimeOutSeconds { get; set; }
+        public int ConnectionTimeOutSeconds { get; set; } = 60;
 
         /// <summary>
         ///     Maximum number of concurrent connections per remote host in cache.
diff --git a/src/Titanium.Web.Proxy/RequestHandler.cs b/src/Titanium.Web.Proxy/RequestHandler.cs
@@ -207,7 +207,7 @@ await clientStreamWriter.WriteResponseAsync(args.WebSession.Response,
                             //for connection pool, retry fails until cache is exhausted.   
                             var result = await retryPolicy<ServerConnectionException>().ExecuteAsync(async (serverConnection) =>
                             {
-                                args.TimeLine["Server Connection Created"] = DateTime.Now;
+                                args.TimeLine["Connection Ready"] = DateTime.Now;
 
                                 // if upgrading to websocket then relay the request without reading the contents
                                 if (request.UpgradeToWebSocket)
@@ -369,6 +369,8 @@ await args.WebSession.SendRequest(Enable100ContinueBehaviour, args.IsTransparent
                 }
             }
 
+            args.TimeLine["Request Sent"] = DateTime.Now;
+
             // If not expectation failed response was returned by server then parse response
             if (!request.ExpectationFailed)
             {
diff --git a/src/Titanium.Web.Proxy/TransparentClientHandler.cs b/src/Titanium.Web.Proxy/TransparentClientHandler.cs
@@ -69,7 +69,7 @@ private async Task handleClient(TransparentProxyEndPoint endPoint, TcpClientConn
                             //it could cause floating server connections when client exits
                             prefetchConnectionTask = tcpConnectionFactory.GetServerConnection(httpsHostName, endPoint.Port,
                                     httpVersion: null, isHttps: true, applicationProtocols: null, isConnect: false,
-                                    proxyServer: this, upStreamEndPoint: UpStreamEndPoint, externalProxy: UpStreamHttpsProxy,
+                                    proxyServer: this, session: null, upStreamEndPoint: UpStreamEndPoint, externalProxy: UpStreamHttpsProxy,
                                     noCache: false, cancellationToken: CancellationToken.None);
                         }
                         
@@ -102,7 +102,7 @@ private async Task handleClient(TransparentProxyEndPoint endPoint, TcpClientConn
                     {
                         var connection = await tcpConnectionFactory.GetServerConnection(httpsHostName, endPoint.Port,
                                     httpVersion: null, isHttps: false, applicationProtocols: null,
-                                    isConnect: true, proxyServer: this, upStreamEndPoint: UpStreamEndPoint,
+                                    isConnect: true, proxyServer: this, session:null, upStreamEndPoint: UpStreamEndPoint,
                                     externalProxy: UpStreamHttpsProxy, noCache: true, cancellationToken: cancellationToken);
 
                         try

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"fileOptions": {`
`3`	`3`	`"excludeSearchPatterns": [`
	`4`	`+ "*/.sln",`
`4`	`5`	`"*/.Docs.csproj",`
`5`	`6`	`"**/tests/",`
`6`	`7`	`"**/Titanium.Web.Proxy.Examples.Wpf/",`
Original file line number	Diff line number	Diff line change
`@@ -207,7 +207,7 @@ await clientStreamWriter.WriteResponseAsync(args.WebSession.Response,`
`207`	`207`	`//for connection pool, retry fails until cache is exhausted.`
`208`	`208`	`var result = await retryPolicy<ServerConnectionException>().ExecuteAsync(async (serverConnection) =>`
`209`	`209`	`{`
`210`		`- args.TimeLine["Server Connection Created"] = DateTime.Now;`
	`210`	`+ args.TimeLine["Connection Ready"] = DateTime.Now;`
`211`	`211`
`212`	`212`	`// if upgrading to websocket then relay the request without reading the contents`
`213`	`213`	`if (request.UpgradeToWebSocket)`
`@@ -369,6 +369,8 @@ await args.WebSession.SendRequest(Enable100ContinueBehaviour, args.IsTransparent`
`369`	`369`	`}`
`370`	`370`	`}`
`371`	`371`
	`372`	`+ args.TimeLine["Request Sent"] = DateTime.Now;`
	`373`	`+`
`372`	`374`	`// If not expectation failed response was returned by server then parse response`
`373`	`375`	`if (!request.ExpectationFailed)`
`374`	`376`	`{`