PoC: low level TLS machine

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs

@@ -874,7 +874,8 @@ internal static int CertVerifyCallback(IntPtr storeCtx, IntPtr arg)
                     .TryGetTarget(out SslAuthenticationOptions? options);
                 Debug.Assert(options != null, "Expected to get SslAuthenticationOptions from GCHandle");
 
-                sslHandle = (SafeSslHandle)options!.SslStream!._securityContext!;
+                sslHandle = options!.SafeSslHandle as SafeSslHandle;
+                Debug.Assert(sslHandle is not null, "Expected SslAuthenticationOptions.SafeSslHandle to be set by SafeSslHandle.Create");
 
                 // We need to note the number of certs in ExtraStore that were
                 // provided (by the user), we will add more from the received peer
@@ -888,7 +889,9 @@ internal static int CertVerifyCallback(IntPtr storeCtx, IntPtr arg)
                 try
                 {
                     ProtocolToken alertToken = default;
-                    if (options.SslStream!.VerifyRemoteCertificate(certificate, chain, options.CertificateContext?.Trust, ref alertToken, out SslPolicyErrors sslPolicyErrors, out X509ChainStatusFlags chainStatus))
+                    SslAuthenticationOptions.VerifyRemoteCertificateCallback? validator = options.RemoteCertificateValidator;
+                    Debug.Assert(validator is not null, "Expected SslAuthenticationOptions.RemoteCertificateValidator to be set by SslStream or TlsSession");
+                    if (validator!(certificate, chain, options.CertificateContext?.Trust, ref alertToken, out SslPolicyErrors sslPolicyErrors, out X509ChainStatusFlags chainStatus))
                     {
                         Ssl.X509StoreCtxSetError(storeCtx, (int)Interop.Crypto.X509VerifyStatusCodeUniversal.X509_V_OK);
                         return 1;

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Ssl.cs

@@ -451,6 +451,10 @@ public static SafeSslHandle Create(SafeSslContextHandle context, SslAuthenticati
             handle._authOptionsHandle = new WeakGCHandle<SslAuthenticationOptions>(options);
             Interop.Ssl.SslSetData(handle, WeakGCHandle<SslAuthenticationOptions>.ToIntPtr(handle._authOptionsHandle));
 
+            // CertVerifyCallback needs the SafeSslHandle to stash a
+            // CertificateValidationException; expose it via the options.
+            options.SafeSslHandle = handle;
+
             // SslSetBio will transfer ownership of the BIO handles to the SSL context
             try
             {

src/libraries/System.Net.Security/ref/System.Net.Security.cs

@@ -687,6 +687,51 @@ public enum TlsCipherSuite : ushort
         TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256 = (ushort)53251,
         TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256 = (ushort)53253,
     }
+    public enum TlsOperationStatus
+    {
+        Complete = 0,
+        WantRead = 1,
+        WantWrite = 2,
+        Closed = 3,
+        WantCredentials = 4,
+        NeedsCertificateValidation = 5,
+    }
+    public sealed partial class TlsContext : System.IDisposable
+    {
+        internal TlsContext() { }
+        public bool IsServer { get { throw null; } }
+        public bool UseExternalCertificateValidation { get { throw null; } set { } }
+        public static System.Net.Security.TlsContext Create(System.Net.Security.SslServerAuthenticationOptions options) { throw null; }
+        public static System.Net.Security.TlsContext Create(System.Net.Security.SslClientAuthenticationOptions options) { throw null; }
+        public void Dispose() { }
+    }
+    public sealed partial class TlsSession : System.IDisposable
+    {
+        internal TlsSession() { }
+        public bool IsServer { get { throw null; } }
+        public bool IsHandshakeComplete { get { throw null; } }
+        public bool HasPendingOutput { get { throw null; } }
+        public string? TargetHostName { get { throw null; } set { } }
+        public System.Security.Authentication.SslProtocols NegotiatedProtocol { get { throw null; } }
+        [System.CLSCompliantAttribute(false)]
+        public System.Net.Security.TlsCipherSuite NegotiatedCipherSuite { get { throw null; } }
+        public System.Net.Security.SslApplicationProtocol NegotiatedApplicationProtocol { get { throw null; } }
+        public static System.Net.Security.TlsSession Create(System.Net.Security.TlsContext context) { throw null; }
+        public System.Net.Security.TlsOperationStatus ProcessHandshake(System.ReadOnlySpan<byte> input, System.Span<byte> output, out int consumed, out int produced) { throw null; }
+        public System.Net.Security.TlsOperationStatus Encrypt(System.ReadOnlySpan<byte> plaintext, System.Span<byte> ciphertext, out int consumed, out int produced) { throw null; }
+        public System.Net.Security.TlsOperationStatus Decrypt(System.ReadOnlySpan<byte> ciphertext, System.Span<byte> plaintext, out int consumed, out int produced) { throw null; }
+        public System.Net.Security.TlsOperationStatus Shutdown(System.Span<byte> ciphertext, out int produced) { throw null; }
+        public System.Net.Security.TlsOperationStatus DrainPendingOutput(System.Span<byte> ciphertext, out int produced) { throw null; }
+        public System.Security.Cryptography.X509Certificates.X509Certificate2? GetRemoteCertificate() { throw null; }
+        public System.Security.Cryptography.X509Certificates.X509Chain? GetRemoteCertificateChain() { throw null; }
+        public System.Net.Security.SslPolicyErrors AcceptWithDefaultValidation() { throw null; }
+        public void SetRemoteCertificateValidationResult(System.Net.Security.SslPolicyErrors errors) { }
+        public System.Security.Cryptography.X509Certificates.X509Certificate2? LocalCertificate { get { throw null; } }
+        public System.Security.Authentication.ExtendedProtection.ChannelBinding? GetChannelBinding(System.Security.Authentication.ExtendedProtection.ChannelBindingKind kind) { throw null; }
+        public System.Net.Security.TlsOperationStatus RequestClientCertificate(System.Span<byte> ciphertext, out int produced) { throw null; }
+        public System.Net.Security.TlsOperationStatus RequestRenegotiation(System.Span<byte> ciphertext, out int produced) { throw null; }
+        public void Dispose() { }
+    }
 }
 namespace System.Security.Authentication
 {

src/libraries/System.Net.Security/src/System.Net.Security.csproj

@@ -62,6 +62,16 @@
     <Compile Include="System\Net\Security\SslStream.cs" />
     <Compile Include="System\Net\Security\SslStream.IO.cs" />
     <Compile Include="System\Net\Security\SslStream.Protocol.cs" />
+    <Compile Include="System\Net\Security\SslStream.TlsSessionWedge.cs"
+             Condition="'$(TargetPlatformIdentifier)' == 'linux' or '$(TargetPlatformIdentifier)' == 'freebsd' or '$(TargetPlatformIdentifier)' == 'windows'" />
+    <Compile Include="System\Net\Security\SslStream.NotUnix.cs"
+             Condition="'$(TargetPlatformIdentifier)' != 'linux' and '$(TargetPlatformIdentifier)' != 'freebsd' and '$(TargetPlatformIdentifier)' != 'windows'" />
+    <Compile Include="System\Net\Security\TlsContext.cs" />
+    <Compile Include="System\Net\Security\TlsOperationStatus.cs" />
+    <Compile Include="System\Net\Security\TlsSession.cs"
+             Condition="'$(TargetPlatformIdentifier)' == 'linux' or '$(TargetPlatformIdentifier)' == 'freebsd' or '$(TargetPlatformIdentifier)' == 'windows'" />
+    <Compile Include="System\Net\Security\TlsSession.Stub.cs"
+             Condition="'$(TargetPlatformIdentifier)' != 'linux' and '$(TargetPlatformIdentifier)' != 'freebsd' and '$(TargetPlatformIdentifier)' != 'windows'" />
     <Compile Include="System\Net\Security\SslStreamCertificateContext.cs" />
     <Compile Include="System\Net\Security\SslConnectionInfo.cs" />
     <Compile Include="System\Net\Security\StreamSizes.cs" />

src/libraries/System.Net.Security/src/System/Net/Security/NetEventSource.Security.cs

@@ -208,32 +208,32 @@ public void SspiSelectedCipherSuite(
 #pragma warning restore SYSLIB0058 // Use NegotiatedCipherSuite.
 
         [NonEvent]
-        public void RemoteCertificateError(SslStream SslStream, string message) =>
-            RemoteCertificateError(GetHashCode(SslStream), message);
+        public void RemoteCertificateError(object sender, string message) =>
+            RemoteCertificateError(GetHashCode(sender), message);
 
         [Event(RemoteCertificateErrorId, Level = EventLevel.Verbose)]
         private void RemoteCertificateError(int sslStreamHash, string message) =>
             WriteEvent(RemoteCertificateErrorId, sslStreamHash, message);
 
         [NonEvent]
-        public void RemoteCertDeclaredValid(SslStream SslStream) =>
-            RemoteCertDeclaredValid(GetHashCode(SslStream));
+        public void RemoteCertDeclaredValid(object sender) =>
+            RemoteCertDeclaredValid(GetHashCode(sender));
 
         [Event(RemoteVertificateValidId, Level = EventLevel.Verbose)]
         private void RemoteCertDeclaredValid(int sslStreamHash) =>
             WriteEvent(RemoteVertificateValidId, sslStreamHash);
 
         [NonEvent]
-        public void RemoteCertHasNoErrors(SslStream SslStream) =>
-            RemoteCertHasNoErrors(GetHashCode(SslStream));
+        public void RemoteCertHasNoErrors(object sender) =>
+            RemoteCertHasNoErrors(GetHashCode(sender));
 
         [Event(RemoteCertificateSuccessId, Level = EventLevel.Verbose)]
         private void RemoteCertHasNoErrors(int sslStreamHash) =>
             WriteEvent(RemoteCertificateSuccessId, sslStreamHash);
 
         [NonEvent]
-        public void RemoteCertUserDeclaredInvalid(SslStream SslStream) =>
-            RemoteCertUserDeclaredInvalid(GetHashCode(SslStream));
+        public void RemoteCertUserDeclaredInvalid(object sender) =>
+            RemoteCertUserDeclaredInvalid(GetHashCode(sender));
 
         [Event(RemoteCertificateInvalidId, Level = EventLevel.Verbose)]
         private void RemoteCertUserDeclaredInvalid(int sslStreamHash) =>

src/libraries/System.Net.Security/src/System/Net/Security/SslAuthenticationOptions.cs

@@ -225,6 +225,25 @@ internal void SetCertificateContextFromCert(X509Certificate2 certificate, bool?
 
 #if !TARGET_WINDOWS && !SYSNETSECURITY_NO_OPENSSL
         internal SslStream? SslStream { get; set; }
+
+        // Set by SafeSslHandle.Create so OpenSSL's CertVerifyCallback can stash
+        // a CertificateValidationException on the handle when validation fails.
+        // Typed as the base SafeHandle so this file compiles in test projects
+        // that don't include the OpenSSL interop sources.
+        internal System.Runtime.InteropServices.SafeHandle? SafeSslHandle { get; set; }
+
+        // Hook invoked by OpenSSL's CertVerifyCallback to drive remote
+        // certificate validation. Set by SslStream and by standalone TlsSession
+        // so both flows share the same callback plumbing.
+        internal delegate bool VerifyRemoteCertificateCallback(
+            X509Certificate2? certificate,
+            X509Chain? chain,
+            SslCertificateTrust? trust,
+            ref ProtocolToken alertToken,
+            out SslPolicyErrors sslPolicyErrors,
+            out X509ChainStatusFlags chainStatus);
+
+        internal VerifyRemoteCertificateCallback? RemoteCertificateValidator { get; set; }
 #endif
 
         public void Dispose()

src/libraries/System.Net.Security/src/System/Net/Security/SslSessionsCache.cs

@@ -114,15 +114,14 @@ public bool Equals(SslCredKey other)
             bool allowRsaPssPadding,
             bool allowRsaPkcs1Padding)
         {
+            var key = new SslCredKey(thumbPrint, (int)sslProtocols, isServer, encryptionPolicy, sendTrustList, checkRevocation, allowTlsResume, allowRsaPssPadding, allowRsaPkcs1Padding);
+
             if (s_cachedCreds.IsEmpty)
             {
                 if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(null, $"Not found, Current Cache Count = {s_cachedCreds.Count}");
                 return null;
             }
 
-            var key = new SslCredKey(thumbPrint, (int)sslProtocols, isServer, encryptionPolicy, sendTrustList, checkRevocation, allowTlsResume, allowRsaPssPadding, allowRsaPkcs1Padding);
-
-            //SafeCredentialReference? cached;
             SafeFreeCredentials? credentials = GetCachedCredential(key);
             if (credentials == null || credentials.IsClosed || credentials.IsInvalid || credentials.Expiry < DateTime.UtcNow)
             {

src/libraries/System.Net.Security/src/System/Net/Security/SslStream.NotUnix.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Net.Security
+{
+    // Stub partial impls for non-Linux/FreeBSD platforms. Always returns false so
+    // SslStream's existing PAL paths run unchanged.
+    public partial class SslStream
+    {
+#pragma warning disable CA1822 // partial method signature must match the Unix impl which is non-static
+        private partial bool TryNextMessageViaTlsSession(ReadOnlySpan<byte> incomingBuffer, out ProtocolToken token, out int consumed)
+        {
+            token = default;
+            consumed = 0;
+            return false;
+        }
+#pragma warning restore CA1822
+    }
+}