[Platform] Detect and fix certificates with potentially inaccessible keys on Mac OS (3.0) (#17580)

* [Platform] Provide a better error message when the developer certificate can't be used (#16659)

Improves the error message Kestrel gives when the developer certificate key is not available for some reason.

* [Platform] Add logic to dotnet-dev-certs to detect and fix certificates with inaccessible keys on Mac OS

* Update the docs link

Author: javiercn

src/Servers/Kestrel/Core/src/CoreStrings.resx

@@ -617,4 +617,7 @@ For more information on configuring HTTPS see https://go.microsoft.com/fwlink/?l
   <data name="Http2TellClientToCalmDown" xml:space="preserve">
     <value>A new stream was refused because this connection has too many streams that haven't finished processing. This may happen if many streams are aborted but not yet cleaned up.</value>
   </data>
+  <data name="BadDeveloperCertificateState" xml:space="preserve">
+    <value>The ASP.NET Core developer certificate is in an invalid state. To fix this issue, run the following commands 'dotnet dev-certs https --clean' and 'dotnet dev-certs https' to remove all existing ASP.NET Core development certificates and create a new untrusted developer certificate. On macOS or Windows, use 'dotnet dev-certs https --trust' to trust the new certificate.</value>
+  </data>
 </root>

src/Servers/Kestrel/Core/src/KestrelServerOptions.cs

@@ -142,8 +142,7 @@ private void EnsureDefaultCert()
                 var logger = ApplicationServices.GetRequiredService<ILogger<KestrelServer>>();
                 try
                 {
-                    var certificateManager = new CertificateManager();
-                    DefaultCertificate = certificateManager.ListCertificates(CertificatePurpose.HTTPS, StoreName.My, StoreLocation.CurrentUser, isValid: true)
+                    DefaultCertificate = CertificateManager.ListCertificates(CertificatePurpose.HTTPS, StoreName.My, StoreLocation.CurrentUser, isValid: true)
                         .FirstOrDefault();
 
                     if (DefaultCertificate != null)

src/Servers/Kestrel/Core/src/Middleware/HttpsConnectionMiddleware.cs

@@ -11,6 +11,7 @@
 using System.Security.Cryptography.X509Certificates;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Certificates.Generation;
 using Microsoft.AspNetCore.Connections;
 using Microsoft.AspNetCore.Connections.Features;
 using Microsoft.AspNetCore.Http.Features;
@@ -208,12 +209,28 @@ private async Task InnerOnConnectionAsync(ConnectionContext context)
                     await sslStream.DisposeAsync();
                     return;
                 }
-                catch (Exception ex) when (ex is IOException || ex is AuthenticationException)
+                catch (IOException ex)
                 {
                     _logger?.LogDebug(1, ex, CoreStrings.AuthenticationFailed);
                     await sslStream.DisposeAsync();
                     return;
                 }
+                catch (AuthenticationException ex)
+                {
+                    if (_serverCertificate == null ||
+                        !CertificateManager.IsHttpsDevelopmentCertificate(_serverCertificate) ||
+                        CertificateManager.CheckDeveloperCertificateKey(_serverCertificate))
+                    {
+                        _logger?.LogDebug(1, ex, CoreStrings.AuthenticationFailed);
+                    }
+                    else
+                    {
+                        _logger?.LogError(2, ex, CoreStrings.BadDeveloperCertificateState);
+                    }
+
+                    await sslStream.DisposeAsync();
+                    return;
+                }
             }
 
             feature.ApplicationProtocol = sslStream.NegotiatedApplicationProtocol.Protocol;

src/Servers/Kestrel/test/InMemory.FunctionalTests/HttpsTests.cs

@@ -364,7 +364,7 @@ public async Task ClientAttemptingToUseUnsupportedProtocolIsLoggedAsDebug()
                 new TestServiceContext(LoggerFactory),
                 listenOptions =>
                 {
-                    listenOptions.UseHttps(TestResources.GetTestCertificate());
+                    listenOptions.UseHttps(TestResources.GetTestCertificate("no_extensions.pfx"));
                 }))
             {
                 using (var connection = server.CreateConnection())
@@ -383,6 +383,35 @@ await Assert.ThrowsAsync<IOException>(() =>
             Assert.Equal(LogLevel.Debug, loggerProvider.FilterLogger.LastLogLevel);
         }
 
+        [Fact]
+        public async Task DevCertWithInvalidPrivateKeyProducesCustomWarning()
+        {
+            var loggerProvider = new HandshakeErrorLoggerProvider();
+            LoggerFactory.AddProvider(loggerProvider);
+
+            await using (var server = new TestServer(context => Task.CompletedTask,
+                new TestServiceContext(LoggerFactory),
+                listenOptions =>
+                {
+                    listenOptions.UseHttps(TestResources.GetTestCertificate());
+                }))
+            {
+                using (var connection = server.CreateConnection())
+                using (var sslStream = new SslStream(connection.Stream, true, (sender, certificate, chain, errors) => true))
+                {
+                    // SslProtocols.Tls is TLS 1.0 which isn't supported by Kestrel by default.
+                    await Assert.ThrowsAsync<IOException>(() =>
+                        sslStream.AuthenticateAsClientAsync("127.0.0.1", clientCertificates: null,
+                            enabledSslProtocols: SslProtocols.Tls,
+                            checkCertificateRevocation: false));
+                }
+            }
+
+            await loggerProvider.FilterLogger.LogTcs.Task.DefaultTimeout();
+            Assert.Equal(2, loggerProvider.FilterLogger.LastEventId);
+            Assert.Equal(LogLevel.Error, loggerProvider.FilterLogger.LastLogLevel);
+        }
+
         [Fact]
         public async Task OnAuthenticate_SeesOtherSettings()
         {

src/Shared/CertificateGeneration/CertificateManager.cs

@@ -41,12 +41,18 @@ internal class CertificateManager
         private const string MacOSTrustCertificateCommandLine = "sudo";
         private static readonly string MacOSTrustCertificateCommandLineArguments = "security add-trusted-cert -d -r trustRoot -k " + MacOSSystemKeyChain + " ";
         private const int UserCancelledErrorCode = 1223;
+        private const string MacOSSetPartitionKeyPermissionsCommandLine = "sudo";
+        private static readonly string MacOSSetPartitionKeyPermissionsCommandLineArguments = "security set-key-partition-list -D localhost -S unsigned:,teamid:UBF8T346G9 " + MacOSUserKeyChain;
 
         // Setting to 0 means we don't append the version byte,
         // which is what all machines currently have.
-        public int AspNetHttpsCertificateVersion { get; set; } = 1;
+        public static int AspNetHttpsCertificateVersion { get; set; } = 1;
 
-        public IList<X509Certificate2> ListCertificates(
+        public static bool IsHttpsDevelopmentCertificate(X509Certificate2 certificate) =>
+            certificate.Extensions.OfType<X509Extension>()
+            .Any(e => string.Equals(AspNetHttpsOid, e.Oid.Value, StringComparison.Ordinal));
+
+        public static IList<X509Certificate2> ListCertificates(
             CertificatePurpose purpose,
             StoreName storeName,
             StoreLocation location,
@@ -173,6 +179,27 @@ private static void DisposeCertificates(IEnumerable<X509Certificate2> disposable
             }
         }
 
+        internal bool HasValidCertificateWithInnaccessibleKeyAcrossPartitions()
+        {
+            var certificates = GetHttpsCertificates();
+            if (certificates.Count == 0)
+            {
+                return false;
+            }
+
+            // We need to check all certificates as a new one might be created that hasn't been correctly setup.
+            var result = false;
+            foreach (var certificate in certificates)
+            {
+                result = result || !CanAccessCertificateKeyAcrossPartitions(certificate);
+            }
+
+            return result;
+        }
+
+        public IList<X509Certificate2> GetHttpsCertificates() =>
+            ListCertificates(CertificatePurpose.HTTPS, StoreName.My, StoreLocation.CurrentUser, isValid: true, requireExportable: true);
+
         public X509Certificate2 CreateAspNetCoreHttpsDevelopmentCertificate(DateTimeOffset notBefore, DateTimeOffset notAfter, string subjectOverride, DiagnosticInformation diagnostics = null)
         {
             var subject = new X500DistinguishedName(subjectOverride ?? LocalhostHttpsDistinguishedName);
@@ -228,6 +255,33 @@ public X509Certificate2 CreateAspNetCoreHttpsDevelopmentCertificate(DateTimeOffs
             return certificate;
         }
 
+        internal static bool CheckDeveloperCertificateKey(X509Certificate2 candidate)
+        {
+            // Tries to use the certificate key to validate it can't access it
+            try
+            {
+                var rsa = candidate.GetRSAPrivateKey();
+                if (rsa == null)
+                {
+                    return false;
+                }
+
+                // Encrypting a random value is the ultimate test for a key validity.
+                // Windows and Mac OS both return HasPrivateKey = true if there is (or there has been) a private key associated
+                // with the certificate at some point.
+                var value = new byte[32];
+                RandomNumberGenerator.Fill(value);
+                rsa.Decrypt(rsa.Encrypt(value, RSAEncryptionPadding.Pkcs1), RSAEncryptionPadding.Pkcs1);
+
+                // Being able to encrypt and decrypt a payload is the strongest guarantee that the key is valid.
+                return true;
+            }
+            catch (Exception)
+            {
+                return false;
+            }
+        }
+
         public X509Certificate2 CreateSelfSignedCertificate(
             X500DistinguishedName subject,
             IEnumerable<X509Extension> extensions,
@@ -676,9 +730,10 @@ public DetailedEnsureCertificateResult EnsureAspNetCoreHttpsDevelopmentCertifica
             bool trust = false,
             bool includePrivateKey = false,
             string password = null,
-            string subject = LocalhostHttpsDistinguishedName)
+            string subject = LocalhostHttpsDistinguishedName,
+            bool isInteractive = true)
         {
-            return EnsureValidCertificateExists(notBefore, notAfter, CertificatePurpose.HTTPS, path, trust, includePrivateKey, password, subject);
+            return EnsureValidCertificateExists(notBefore, notAfter, CertificatePurpose.HTTPS, path, trust, includePrivateKey, password, subject, isInteractive);
         }
 
         public DetailedEnsureCertificateResult EnsureValidCertificateExists(
@@ -689,7 +744,8 @@ public DetailedEnsureCertificateResult EnsureValidCertificateExists(
             bool trust,
             bool includePrivateKey,
             string password,
-            string subject)
+            string subject,
+            bool isInteractive)
         {
             if (purpose == CertificatePurpose.All)
             {
@@ -716,6 +772,35 @@ public DetailedEnsureCertificateResult EnsureValidCertificateExists(
                 result.Diagnostics.Debug("Skipped filtering certificates by subject.");
             }
 
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                foreach (var cert in filteredCertificates)
+                {
+                    if (!CanAccessCertificateKeyAcrossPartitions(cert))
+                    {
+                        if (!isInteractive)
+                        {
+                            // If the process is not interactive (first run experience) bail out. We will simply create a certificate
+                            // in case there is none or report success during the first run experience.
+                            break;
+                        }
+                        try
+                        {
+                            // The command we run handles making keys for all localhost certificates accessible across partitions. If it can not run the
+                            // command safely (because there are other localhost certificates that were not created by asp.net core, it will throw.
+                            MakeCertificateKeyAccessibleAcrossPartitions(cert);
+                            break;
+                        }
+                        catch (Exception ex)
+                        {
+                            result.Diagnostics.Error("Failed to make certificate key accessible", ex);
+                            result.ResultCode = EnsureCertificateResult.FailedToMakeKeyAccessible;
+                            return result;
+                        }
+                    }
+                }
+            }
+
             certificates = filteredCertificates;
 
             result.ResultCode = EnsureCertificateResult.Succeeded;
@@ -763,6 +848,11 @@ public DetailedEnsureCertificateResult EnsureValidCertificateExists(
                     result.ResultCode = EnsureCertificateResult.ErrorSavingTheCertificateIntoTheCurrentUserPersonalStore;
                     return result;
                 }
+
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) && isInteractive)
+                {
+                    MakeCertificateKeyAccessibleAcrossPartitions(certificate);
+                }
             }
             if (path != null)
             {
@@ -804,6 +894,73 @@ public DetailedEnsureCertificateResult EnsureValidCertificateExists(
             return result;
         }
 
+        private void MakeCertificateKeyAccessibleAcrossPartitions(X509Certificate2 certificate)                {
+            if (OtherNonAspNetCoreHttpsCertificatesPresent())
+            {
+                throw new InvalidOperationException("Unable to make HTTPS ceritificate key trusted across security partitions.");
+            }
+            using (var process = Process.Start(MacOSSetPartitionKeyPermissionsCommandLine, MacOSSetPartitionKeyPermissionsCommandLineArguments))
+            {
+                process.WaitForExit();
+                if (process.ExitCode != 0)
+                {
+                    throw new InvalidOperationException("Error making the key accessible across partitions.");
+                }
+            }
+
+            var certificateSentinelPath = GetCertificateSentinelPath(certificate);
+            File.WriteAllText(certificateSentinelPath, "true");
+        }
+
+        private static string GetCertificateSentinelPath(X509Certificate2 certificate) =>
+            Path.Combine(Environment.GetEnvironmentVariable("HOME"), ".dotnet", $"certificate.{certificate.GetCertHashString(HashAlgorithmName.SHA256)}.sentinel");
+
+        private bool OtherNonAspNetCoreHttpsCertificatesPresent()
+        {
+            var certificates = new List<X509Certificate2>();
+            try
+            {
+                using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser))
+                {
+                    store.Open(OpenFlags.ReadOnly);
+                    certificates.AddRange(store.Certificates.OfType<X509Certificate2>());
+                    IEnumerable<X509Certificate2> matchingCertificates = certificates;
+                        // Ensure the certificate hasn't expired, has a private key and its exportable
+                        // (for container/unix scenarios).
+                        var now = DateTimeOffset.Now;
+                        matchingCertificates = matchingCertificates
+                            .Where(c => c.NotBefore <= now &&
+                                now <= c.NotAfter && c.Subject == LocalhostHttpsDistinguishedName);
+
+                    // We need to enumerate the certificates early to prevent dispoisng issues.
+                    matchingCertificates = matchingCertificates.ToList();
+
+                    var certificatesToDispose = certificates.Except(matchingCertificates);
+                    DisposeCertificates(certificatesToDispose);
+
+                    store.Close();
+
+                    return matchingCertificates.All(c => !HasOid(c, AspNetHttpsOid));
+                }
+            }
+            catch
+            {
+                DisposeCertificates(certificates);
+                certificates.Clear();
+                return true;
+            }
+
+            bool HasOid(X509Certificate2 certificate, string oid) =>
+                certificate.Extensions.OfType<X509Extension>()
+                .Any(e => string.Equals(oid, e.Oid.Value, StringComparison.Ordinal));
+        }
+
+        private bool CanAccessCertificateKeyAcrossPartitions(X509Certificate2 certificate)
+        {
+            var certificateSentinelPath = GetCertificateSentinelPath(certificate);
+            return File.Exists(certificateSentinelPath);
+        }
+
         private class UserCancelledTrustException : Exception
         {
         }

src/Shared/CertificateGeneration/EnsureCertificateResult.cs

@@ -11,6 +11,7 @@ internal enum EnsureCertificateResult
         ErrorSavingTheCertificateIntoTheCurrentUserPersonalStore,
         ErrorExportingTheCertificate,
         FailedToTrustTheCertificate,
-        UserCancelledTrustStep
+        UserCancelledTrustStep,
+        FailedToMakeKeyAccessible,
     }
 }

src/Tools/FirstRunCertGenerator/src/CertificateGenerator.cs

@@ -9,7 +9,7 @@ public static void GenerateAspNetHttpsCertificate()
         {
             var manager = new CertificateManager();
             var now = DateTimeOffset.Now;
-            manager.EnsureAspNetCoreHttpsDevelopmentCertificate(now, now.AddYears(1));
+            manager.EnsureAspNetCoreHttpsDevelopmentCertificate(now, now.AddYears(1), isInteractive: false);
         }
     }
 }