Support attaching to an existing request queue in HTTP.SYS

src/Servers/HttpSys/HttpSysServer.sln

@@ -70,6 +70,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Hostin
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Connections.Abstractions", "..\Connections.Abstractions\src\Microsoft.AspNetCore.Connections.Abstractions.csproj", "{00A88B8D-D539-45DD-B071-1E955AF89A4A}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QueueSharing", "samples\QueueSharing\QueueSharing.csproj", "{9B58DF76-DC6D-4728-86B7-40087BDDC897}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -304,6 +306,18 @@ Global
 		{00A88B8D-D539-45DD-B071-1E955AF89A4A}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{00A88B8D-D539-45DD-B071-1E955AF89A4A}.Release|x86.ActiveCfg = Release|Any CPU
 		{00A88B8D-D539-45DD-B071-1E955AF89A4A}.Release|x86.Build.0 = Release|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Debug|x86.Build.0 = Debug|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Release|x86.ActiveCfg = Release|Any CPU
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -329,6 +343,7 @@ Global
 		{19DC60DE-C413-43A2-985E-0D0F20AD2302} = {4DA3C456-5050-4AC0-A554-795F6DEC8660}
 		{D93575B3-BFA3-4523-B060-D268D6A0A66B} = {4DA3C456-5050-4AC0-A554-795F6DEC8660}
 		{00A88B8D-D539-45DD-B071-1E955AF89A4A} = {4DA3C456-5050-4AC0-A554-795F6DEC8660}
+		{9B58DF76-DC6D-4728-86B7-40087BDDC897} = {3A1E31E3-2794-4CA3-B8E2-253E96BDE514}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {34B42B42-FA09-41AB-9216-14073990C504}

src/Servers/HttpSys/ref/Microsoft.AspNetCore.Server.HttpSys.netcoreapp.cs

@@ -52,6 +52,7 @@ public HttpSysOptions() { }
         public long? MaxConnections { get { throw null; } set { } }
         public long? MaxRequestBodySize { get { throw null; } set { } }
         public long RequestQueueLimit { get { throw null; } set { } }
+        public Microsoft.AspNetCore.Server.HttpSys.RequestQueueMode RequestQueueMode { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
         public string RequestQueueName { get { throw null; } set { } }
         public bool ThrowWriteExceptions { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
         public Microsoft.AspNetCore.Server.HttpSys.TimeoutManager Timeouts { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
@@ -61,6 +62,12 @@ public partial interface IHttpSysRequestInfoFeature
     {
         System.Collections.Generic.IReadOnlyDictionary<int, System.ReadOnlyMemory<byte>> RequestInfo { get; }
     }
+    public enum RequestQueueMode
+    {
+        Create = 0,
+        Attach = 1,
+        CreateOrAttach = 2,
+    }
     public sealed partial class TimeoutManager
     {
         internal TimeoutManager() { }

src/Servers/HttpSys/samples/QueueSharing/Program.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Xml.Schema;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Server.HttpSys;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace QueueSharing
+{
+    public static class Program
+    {
+        public static void Main(string[] args)
+        {
+            Console.Write("Create and (c)reate, (a)ttach to existing, or attach (o)r create? ");
+            var key = Console.ReadKey();
+            Console.WriteLine();
+            var mode = RequestQueueMode.Create;
+            switch (key.KeyChar)
+            {
+                case 'a':
+                    mode = RequestQueueMode.Attach;
+                    break;
+                case 'o':
+                    mode = RequestQueueMode.CreateOrAttach;
+                    break;
+                case 'c':
+                    mode = RequestQueueMode.Create;
+                    break;
+                default:
+                    Console.WriteLine("Unknown option, defaulting to (c)create.");
+                    break;
+            }
+
+            var host = new HostBuilder()
+                .ConfigureLogging(factory => factory.AddConsole())
+                .ConfigureWebHost(webHost =>
+                {
+                    webHost.UseHttpSys(options =>
+                    {
+                        // Skipping this to ensure the server works without any prefixes in attach mode.
+                        if (mode != RequestQueueMode.Attach)
+                        {
+                            options.UrlPrefixes.Add("http://localhost:5002");
+                        }
+                        options.RequestQueueName = "QueueName";
+                        options.RequestQueueMode = mode;
+                        options.MaxAccepts = 1; // Better load rotation between instances.
+                    }).ConfigureServices(services =>
+                    {
+
+                    }).Configure(app =>
+                    {
+                        app.Run(async context =>
+                        {
+                            context.Response.ContentType = "text/plain";
+                            // There's a strong connection affinity between processes. Close the connection so the next request can be dispatched to a random instance.
+                            // It appears to be round robin based and switch instances roughly every 30 requests when using new connections.
+                            // This seems related to the default MaxAccepts (5 * processor count).
+                            // I'm told this connection affinity does not apply to HTTP/2.
+                            context.Response.Headers["Connection"] = "close";
+                            await context.Response.WriteAsync("Hello world from " + context.Request.Host + " at " + DateTime.Now);
+                        });
+                    });
+
+                })
+                .Build();
+
+            host.Run();
+        }
+    }
+}

src/Servers/HttpSys/samples/QueueSharing/QueueSharing.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
+    <OutputType>Exe</OutputType>
+    <ServerGarbageCollection>true</ServerGarbageCollection>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Reference Include="Microsoft.AspNetCore.Server.HttpSys" />
+    <Reference Include="Microsoft.Extensions.Hosting" />
+    <Reference Include="Microsoft.Extensions.Logging.Console" />
+  </ItemGroup>
+</Project>

src/Servers/HttpSys/src/AuthenticationManager.cs

@@ -32,6 +32,9 @@ internal AuthenticationManager()
         {
         }
 
+        /// <summary>
+        /// When attaching to an existing queue this setting must match the one used to create the queue.
+        /// </summary>
         public AuthenticationSchemes Schemes
         {
             get { return _authSchemes; }

src/Servers/HttpSys/src/HttpSysListener.cs

@@ -69,7 +69,7 @@ public HttpSysListener(HttpSysOptions options, ILoggerFactory loggerFactory)
             // V2 initialization sequence:
             // 1. Create server session
             // 2. Create url group
-            // 3. Create request queue - Done in Start()
+            // 3. Create request queue
             // 4. Add urls to url group - Done in Start()
             // 5. Attach request queue to url group - Done in Start()
 
@@ -79,7 +79,7 @@ public HttpSysListener(HttpSysOptions options, ILoggerFactory loggerFactory)
 
                 _urlGroup = new UrlGroup(_serverSession, Logger);
 
-                _requestQueue = new RequestQueue(_urlGroup, options.RequestQueueName, Logger);
+                _requestQueue = new RequestQueue(_urlGroup, options.RequestQueueName, options.RequestQueueMode, Logger);
 
                 _disconnectListener = new DisconnectListener(_requestQueue, Logger);
             }
@@ -147,20 +147,24 @@ public void Start()
                         return;
                     }
 
-                    Options.Apply(UrlGroup, RequestQueue);
+                    // If this instance created the queue then configure it.
+                    if (_requestQueue.Created)
+                    {
+                        Options.Apply(UrlGroup, RequestQueue);
 
-                    _requestQueue.AttachToUrlGroup();
+                        _requestQueue.AttachToUrlGroup();
 
-                    // All resources are set up correctly. Now add all prefixes.
-                    try
-                    {
-                        Options.UrlPrefixes.RegisterAllPrefixes(UrlGroup);
-                    }
-                    catch (HttpSysException)
-                    {
-                        // If an error occurred while adding prefixes, free all resources allocated by previous steps.
-                        _requestQueue.DetachFromUrlGroup();
-                        throw;
+                        // All resources are set up correctly. Now add all prefixes.
+                        try
+                        {
+                            Options.UrlPrefixes.RegisterAllPrefixes(UrlGroup);
+                        }
+                        catch (HttpSysException)
+                        {
+                            // If an error occurred while adding prefixes, free all resources allocated by previous steps.
+                            _requestQueue.DetachFromUrlGroup();
+                            throw;
+                        }
                     }
 
                     _state = State.Started;
@@ -188,11 +192,15 @@ private void Stop()
                         return;
                     }
 
-                    Options.UrlPrefixes.UnregisterAllPrefixes();
+                    // If this instance created the queue then remove the URL prefixes before shutting down.
+                    if (_requestQueue.Created)
+                    {
+                        Options.UrlPrefixes.UnregisterAllPrefixes();
+                        _requestQueue.DetachFromUrlGroup();
+                    }
 
                     _state = State.Stopped;
 
-                    _requestQueue.DetachFromUrlGroup();
                 }
             }
             catch (Exception exception)

src/Servers/HttpSys/src/HttpSysOptions.cs

@@ -38,7 +38,7 @@ public string RequestQueueName
             get => _requestQueueName;
             set
             {
-                if (value.Length > MaximumRequestQueueNameLength)
+                if (value?.Length > MaximumRequestQueueNameLength)
                 {
                     throw new ArgumentOutOfRangeException(nameof(value),
                                                           value,
@@ -48,6 +48,12 @@ public string RequestQueueName
             }
         }
 
+        /// <summary>
+        /// Indicates if this server instance is responsible for creating and configuring the request queue,
+        /// of if it should attach to an existing queue. The default is to create.
+        /// </summary>
+        public RequestQueueMode RequestQueueMode { get; set; }
+
         /// <summary>
         /// The maximum number of concurrent accepts.
         /// </summary>
@@ -63,6 +69,7 @@ public string RequestQueueName
         /// <summary>
         /// The url prefixes to register with Http.Sys. These may be modified at any time prior to disposing
         /// the listener.
+        /// When attached to an existing queue the prefixes are only used to compute PathBase for requests.
         /// </summary>
         public UrlPrefixCollection UrlPrefixes { get; } = new UrlPrefixCollection();
 
@@ -75,6 +82,7 @@ public string RequestQueueName
         /// <summary>
         /// Exposes the Http.Sys timeout configurations.  These may also be configured in the registry.
         /// These may be modified at any time prior to disposing the listener.
+        /// These settings do not apply when attaching to an existing queue.
         /// </summary>
         public TimeoutManager Timeouts { get; } = new TimeoutManager();
 
@@ -87,6 +95,7 @@ public string RequestQueueName
         /// <summary>
         /// Gets or sets the maximum number of concurrent connections to accept, -1 for infinite, or null to
         /// use the machine wide setting from the registry. The default value is null.
+        /// This settings does not apply when attaching to an existing queue.
         /// </summary>
         public long? MaxConnections
         {
@@ -109,6 +118,7 @@ public long? MaxConnections
 
         /// <summary>
         /// Gets or sets the maximum number of requests that will be queued up in Http.Sys.
+        /// This settings does not apply when attaching to an existing queue.
         /// </summary>
         public long RequestQueueLimit
         {
@@ -164,6 +174,7 @@ public long? MaxRequestBodySize
         /// Gets or sets a value that controls how http.sys reacts when rejecting requests due to throttling conditions - like when the request
         /// queue limit is reached. The default in http.sys is "Basic" which means http.sys is just resetting the TCP connection. IIS uses Limited
         /// as its default behavior which will result in sending back a 503 - Service Unavailable back to the client.
+        /// This settings does not apply when attaching to an existing queue.
         /// </summary>
         public Http503VerbosityLevel Http503Verbosity
         {
@@ -192,6 +203,7 @@ public Http503VerbosityLevel Http503Verbosity
             }
         }
 
+        // Not called when attaching to an existing queue.
         internal void Apply(UrlGroup urlGroup, RequestQueue requestQueue)
         {
             _urlGroup = urlGroup;

src/Servers/HttpSys/src/MessagePump.cs

@@ -112,13 +112,14 @@ public Task StartAsync<TContext>(IHttpApplication<TContext> application, Cancell
                     Listener.Options.UrlPrefixes.Add(value);
                 }
             }
-            else
+            else if (Listener.RequestQueue.Created)
             {
                 LogHelper.LogDebug(_logger, $"No listening endpoints were configured. Binding to {Constants.DefaultServerAddress} by default.");
 
                 _serverAddresses.Addresses.Add(Constants.DefaultServerAddress);
                 Listener.Options.UrlPrefixes.Add(Constants.DefaultServerAddress);
             }
+            // else // Attaching to an existing queue, don't add a default.
 
             // Can't call Start twice
             Contract.Assert(_application == null);

src/Servers/HttpSys/src/NativeInterop/HttpApi.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -65,7 +65,7 @@ internal static unsafe class HttpApi
 
         [DllImport(HTTPAPI, ExactSpelling = true, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, SetLastError = true)]
         internal static extern unsafe uint HttpCreateRequestQueue(HTTPAPI_VERSION version, string pName,
-            UnsafeNclNativeMethods.SECURITY_ATTRIBUTES pSecurityAttributes, uint flags, out HttpRequestQueueV2Handle pReqQueueHandle);
+            UnsafeNclNativeMethods.SECURITY_ATTRIBUTES pSecurityAttributes, HTTP_CREATE_REQUEST_QUEUE_FLAG flags, out HttpRequestQueueV2Handle pReqQueueHandle);
 
         [DllImport(HTTPAPI, ExactSpelling = true, CallingConvention = CallingConvention.StdCall, SetLastError = true)]
         internal static extern unsafe uint HttpCloseRequestQueue(IntPtr pReqQueueHandle);