Add low-level TLS state machine API (TlsContext / TlsSession)

docs/project/list-of-diagnostics.md

@@ -320,3 +320,4 @@ Diagnostic id values for experimental APIs must not be recycled, as that could s
 |  __`SYSLIB5004`__ |     .NET 9 |     TBD | `X86Base.DivRem` is experimental since performance is not as optimized as `T.DivRem` |
 |  __`SYSLIB5005`__ |     .NET 9 | .NET 10 | `System.Formats.Nrbf` is experimental |
 |  __`SYSLIB5006`__ |    .NET 10 |     TBD | Types for Post-Quantum Cryptography (PQC) are experimental. |
+|  __`SYSLIB5007`__ |    .NET 11 |     TBD | Low-level TLS engine types (`TlsContext`, `TlsSession`) in `System.Net.Security` are experimental. |

src/libraries/Common/src/Interop/OSX/Interop.NetworkFramework.Tls.cs

@@ -28,7 +28,7 @@ internal static unsafe partial bool Init(
 
             // Create a new connection context
             [LibraryImport(Interop.Libraries.AppleCryptoNative, EntryPoint = "AppleCryptoNative_NwConnectionCreate", StringMarshalling = StringMarshalling.Utf8)]
-            internal static unsafe partial SafeNwHandle NwConnectionCreate([MarshalAs(UnmanagedType.I4)] bool isServer, IntPtr context, string targetName, byte* alpnBuffer, int alpnLength, SslProtocols minTlsProtocol, SslProtocols maxTlsProtocol, uint* cipherSuites, int cipherSuitesLength);
+            internal static unsafe partial SafeNwHandle NwConnectionCreate([MarshalAs(UnmanagedType.I4)] bool isServer, IntPtr context, string targetName, byte* alpnBuffer, int alpnLength, SslProtocols minTlsProtocol, SslProtocols maxTlsProtocol, uint* cipherSuites, int cipherSuitesLength, IntPtr serverIdentity);
 
             // Start the TLS handshake, notifications are received via the status callback (potentially from a different thread).
             [LibraryImport(Interop.Libraries.AppleCryptoNative, EntryPoint = "AppleCryptoNative_NwConnectionStart")]

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Ssl.cs

@@ -6,6 +6,7 @@
 using System.Collections.ObjectModel;
 using System.Diagnostics;
 using System.Net.Security;
+using System.Net.Sockets;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Runtime.InteropServices.Marshalling;
@@ -117,6 +118,15 @@ internal static unsafe ushort[] GetDefaultSignatureAlgorithms()
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslSetBio")]
         internal static partial void SslSetBio(SafeSslHandle ssl, SafeBioHandle rbio, SafeBioHandle wbio);
 
+        // The OpenSSL shims below report errors via out params + the OpenSSL error
+        // queue (SSL_get_error / ERR_get_error); they do not set errno. SetLastError is
+        // omitted so we don't pay the marshaller cost of capturing a value no caller reads.
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslSetFd")]
+        internal static partial int SslSetFd(SafeSslHandle ssl, SafeSocketHandle socket);
+
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslDoHandshake")]
+        internal static partial int SslDoHandshake(SafeSslHandle ssl, out SslErrorCode error);
+
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslHandshake", SetLastError = true)]
         internal static unsafe partial int SslHandshake(
             SafeSslHandle ssl,
@@ -166,6 +176,38 @@ internal static unsafe partial int SslDecrypt(
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioNewManagedSpan")]
         internal static partial SafeBioHandle BioNewManagedSpan();
 
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioNewSocketReplay")]
+        private static unsafe partial SafeBioHandle BioNewSocketReplay(IntPtr fd, byte* prefix, int prefixLen);
+
+        internal static unsafe SafeBioHandle BioNewSocketReplay(SafeSocketHandle socket, ReadOnlySpan<byte> prefix)
+        {
+            fixed (byte* pPrefix = prefix)
+            {
+                return BioNewSocketReplay(socket.DangerousGetHandle(), pPrefix, prefix.Length);
+            }
+        }
+
+        // Reads directly from the BIO's bound fd into its internal peek buffer until a
+        // full TLS record is present. Returns:
+        //   1  = have full frame; framePtr / frameLen point into the BIO's buffer.
+        //   0  = need more data (fd would block); caller polls SelectRead and retries.
+        //  -1  = error (EOF, oversized record, or recv failure).
+        //
+        // The returned pointer is valid until the BIO is destroyed or SocketReplayBioRead
+        // starts consuming the buffer (i.e. once SSL_do_handshake runs against this BIO).
+        // Callers must span-wrap and parse before creating the SSL* that owns the BIO.
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioReadTlsFrame")]
+        internal static unsafe partial int BioReadTlsFrame(SafeBioHandle bio, out byte* framePtr, out int frameLen);
+
+        // Returns the socket-replay BIO's retained peek buffer (bytes captured by
+        // BioReadTlsFrame). Valid until the BIO is destroyed, even after OpenSSL has
+        // drained it during handshake.
+        //   1  = prefix present; prefixPtr / prefixLen wrap the internal buffer.
+        //   0  = BIO has no captured prefix.
+        //  -1  = error (invalid args).
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioGetPrefix")]
+        internal static unsafe partial int BioGetPrefix(SafeBioHandle bio, out byte* prefixPtr, out int prefixLen);
+
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_BioGetWriteResult")]
         internal static partial void BioGetWriteResult(SafeBioHandle bio, out int writtenToWindow, out int spillLen);
 
@@ -231,6 +273,9 @@ internal static SafeSharedX509StackHandle SslGetPeerCertChain(SafeSslHandle ssl)
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslSetVerifyPeer")]
         internal static partial void SslSetVerifyPeer(SafeSslHandle ssl, [MarshalAs(UnmanagedType.Bool)] bool failIfNoPeerCert);
 
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslSetRetryVerify")]
+        internal static partial int SslSetRetryVerify(SafeSslHandle ssl);
+
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslGetData")]
         internal static partial IntPtr SslGetData(IntPtr ssl);
 
@@ -422,6 +467,7 @@ internal enum SslErrorCode
             SSL_ERROR_WANT_X509_LOOKUP = 4,
             SSL_ERROR_SYSCALL = 5,
             SSL_ERROR_ZERO_RETURN = 6,
+            SSL_ERROR_WANT_RETRY_VERIFY = 12,
 
             // NOTE: this SslErrorCode value doesn't exist in OpenSSL, but
             // we use it to distinguish when a renegotiation is pending.
@@ -450,6 +496,17 @@ internal sealed class SafeSslHandle : SafeDeleteSslContext
         // we may rethrow it after returning to managed code.
         public Exception? CertificateValidationException;
 
+        // OpenSSL 3.0+ retry-verify state (dormant infrastructure). When CertVerifyCallback
+        // eventually opts into SSL_set_retry_verify (currently disabled — upstream OpenSSL
+        // does not re-enter the peer-cert verify callback on either client or server SSLs),
+        // it will set RetryVerifyAttempted = true and, on re-entry, honor
+        // ExternalValidationAccepted (posted by TlsSession.PushExternalValidationVerdict-
+        // ToPalIfRetryVerify). Both fields are read but never written today, hence CS0649.
+#pragma warning disable CS0649
+        public bool RetryVerifyAttempted;
+        public bool ExternalValidationAccepted;
+#pragma warning restore CS0649
+
         public bool IsServer
         {
             get { return _isServer; }
@@ -478,37 +535,85 @@ internal void MarkHandshakeCompleted()
 
         public static SafeSslHandle Create(SafeSslContextHandle context, SslAuthenticationOptions options)
         {
-            SafeBioHandle readBio = Interop.Ssl.BioNewManagedSpan();
-            SafeBioHandle writeBio = Interop.Ssl.BioNewManagedSpan();
+            SafeSocketHandle? socket = options.SocketHandle;
+            bool useFd = socket is not null && !socket.IsInvalid;
+            SafeBioHandle? preallocatedReadBio = useFd ? options.PreallocatedReadBio : null;
+            byte[]? replayPrefix = useFd ? options.ReplayPrefix : null;
+            bool usePreallocatedBio = preallocatedReadBio is not null;
+            bool useReplayBio = usePreallocatedBio || (useFd && replayPrefix is not null);
+
+            SafeBioHandle? readBio = null;
+            SafeBioHandle? writeBio = null;
+            if (usePreallocatedBio)
+            {
+                // Deferred-server flow (native pre-fetch): the caller populated a
+                // socket-replay BIO via BioReadTlsFrame; adopt it as the read BIO
+                // and create a peer write BIO for OpenSSL's outbound records.
+                // Clear the field so ownership transfer happens exactly once.
+                readBio = preallocatedReadBio;
+                options.PreallocatedReadBio = null;
+                writeBio = Interop.Ssl.BioNewSocketReplay(socket!, ReadOnlySpan<byte>.Empty);
+            }
+            else if (useReplayBio)
+            {
+                // Legacy deferred-server flow (managed pre-fetch): install a socket-
+                // replay BIO seeded with the peeked ClientHello bytes.
+                readBio = Interop.Ssl.BioNewSocketReplay(socket!, replayPrefix);
+                writeBio = Interop.Ssl.BioNewSocketReplay(socket!, ReadOnlySpan<byte>.Empty);
+            }
+            else if (!useFd)
+            {
+                readBio = Interop.Ssl.BioNewManagedSpan();
+                writeBio = Interop.Ssl.BioNewManagedSpan();
+            }
+
             SafeSslHandle handle = Interop.Ssl.SslCreate(context);
-            if (readBio.IsInvalid || writeBio.IsInvalid || handle.IsInvalid)
+            if (((readBio is not null) && (readBio.IsInvalid || writeBio!.IsInvalid)) || handle.IsInvalid)
             {
-                readBio.Dispose();
-                writeBio.Dispose();
+                readBio?.Dispose();
+                writeBio?.Dispose();
                 handle.Dispose(); // will make IsInvalid==true if it's not already
                 return handle;
             }
             handle._isServer = options.IsServer;
             handle._authOptionsHandle = new WeakGCHandle<SslAuthenticationOptions>(options);
             Interop.Ssl.SslSetData(handle, WeakGCHandle<SslAuthenticationOptions>.ToIntPtr(handle._authOptionsHandle));
 
-            // SslSetBio will transfer ownership of the BIO handles to the SSL context
-            try
+            // CertVerifyCallback needs the SafeSslHandle to stash a
+            // CertificateValidationException; expose it via the options.
+            options.SafeSslHandle = handle;
+
+            if (useFd && !useReplayBio)
             {
-                readBio.TransferOwnershipToParent(handle);
-                writeBio.TransferOwnershipToParent(handle);
-                handle._readBio = readBio;
-                handle._writeBio = writeBio;
-                Interop.Ssl.SslSetBio(handle, readBio, writeBio);
+                if (Interop.Ssl.SslSetFd(handle, socket!) != 1)
+                {
+                    handle.Dispose();
+                    throw Interop.OpenSsl.CreateSslException(SR.net_allocate_ssl_context_failed);
+                }
             }
-            catch (Exception exc)
+            else
             {
-                // The only way this should be able to happen without thread aborts is if we hit OOMs while
-                // manipulating the safe handles, in which case we may leak the bio handles.
-                Debug.Fail("Unexpected exception while transferring SafeBioHandle ownership to SafeSslHandle", exc.ToString());
-                throw;
+                // SslSetBio will transfer ownership of the BIO handles to the SSL context
+                try
+                {
+                    readBio!.TransferOwnershipToParent(handle);
+                    writeBio!.TransferOwnershipToParent(handle);
+                    handle._readBio = readBio;
+                    handle._writeBio = writeBio;
+                    Interop.Ssl.SslSetBio(handle, readBio, writeBio);
+                }
+                catch (Exception exc)
+                {
+                    // The only way this should be able to happen without thread aborts is if we hit OOMs while
+                    // manipulating the safe handles, in which case we may leak the bio handles.
+                    Debug.Fail("Unexpected exception while transferring SafeBioHandle ownership to SafeSslHandle", exc.ToString());
+                    throw;
+                }
             }
 
+            // Consumed exactly once: the BIO holds its own copy of the prefix bytes.
+            options.ReplayPrefix = null;
+
             if (options.IsServer)
             {
                 Interop.Ssl.SslSetAcceptState(handle);

src/libraries/Common/src/System/Experimentals.cs

@@ -33,6 +33,9 @@ internal static class Experimentals
         // Types for Post-Quantum Cryptography (PQC) are experimental.
         internal const string PostQuantumCryptographyDiagId = "SYSLIB5006";
 
+        // Low-level TLS engine (TlsContext / TlsSession) is experimental.
+        internal const string LowLevelTlsDiagId = "SYSLIB5007";
+
         // When adding a new diagnostic ID, add it to the table in docs\project\list-of-diagnostics.md as well.
         // Keep new const identifiers above this comment.
     }

src/libraries/System.Net.Security/ref/System.Net.Security.cs

@@ -687,6 +687,72 @@ public enum TlsCipherSuite : ushort
         TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256 = (ushort)53251,
         TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256 = (ushort)53253,
     }
+    [System.Diagnostics.CodeAnalysis.ExperimentalAttribute("SYSLIB5007", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")]
+    public enum TlsOperationStatus
+    {
+        Complete = 0,
+        DestinationTooSmall = 1,
+        NeedMoreData = 2,
+        Closed = 3,
+        CertificateRequested = 4,
+        NeedsCertificateValidation = 5,
+        NeedsTlsContext = 6,
+    }
+    [System.Diagnostics.CodeAnalysis.ExperimentalAttribute("SYSLIB5007", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")]
+    public sealed partial class TlsContext : System.IDisposable
+    {
+        internal TlsContext() { }
+        public static System.Net.Security.TlsContext CreateServer(System.Net.Security.SslServerAuthenticationOptions options) { throw null; }
+        public static System.Net.Security.TlsContext CreateClient(System.Net.Security.SslClientAuthenticationOptions options) { throw null; }
+        public void Dispose() { }
+    }
+    [System.Diagnostics.CodeAnalysis.ExperimentalAttribute("SYSLIB5007", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")]
+    public abstract partial class TlsSession : System.IDisposable
+    {
+        private protected TlsSession() { }
+        public bool IsHandshakeComplete { get { throw null; } }
+        public bool HasPendingOutput { get { throw null; } }
+        public string? TargetHostName { get { throw null; } set { } }
+        public System.Net.Security.SslClientHelloInfo? ClientHelloInfo { get { throw null; } }
+        public int GetClientHelloLength() { throw null; }
+        public bool TryGetClientHelloBytes(System.Span<byte> destination, out int bytesWritten) { throw null; }
+        public System.Security.Authentication.SslProtocols NegotiatedProtocol { get { throw null; } }
+        [System.CLSCompliantAttribute(false)]
+        public System.Net.Security.TlsCipherSuite NegotiatedCipherSuite { get { throw null; } }
+        public System.Net.Security.SslApplicationProtocol NegotiatedApplicationProtocol { get { throw null; } }
+        public System.Security.Cryptography.X509Certificates.X509Certificate2? GetRemoteCertificate() { throw null; }
+        public System.Security.Cryptography.X509Certificates.X509Certificate2Collection? GetRemoteCertificates() { throw null; }
+        public System.Net.Security.SslPolicyErrors AcceptWithDefaultValidation() { throw null; }
+        public void SetRemoteCertificateValidationResult(System.Net.Security.SslPolicyErrors errors) { }
+        public void SetContext(System.Net.Security.TlsContext context) { }
+        public void SetClientCertificateContext(System.Net.Security.SslStreamCertificateContext? context) { }
+        public System.Collections.Generic.IReadOnlyList<string>? GetAcceptableIssuers() { throw null; }
+        public System.Security.Cryptography.X509Certificates.X509Certificate2? LocalCertificate { get { throw null; } }
+        public System.Security.Authentication.ExtendedProtection.ChannelBinding? GetChannelBinding(System.Security.Authentication.ExtendedProtection.ChannelBindingKind kind) { throw null; }
+        public void Dispose() { }
+    }
+    [System.Diagnostics.CodeAnalysis.ExperimentalAttribute("SYSLIB5007", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")]
+    public sealed partial class TlsBufferSession : System.Net.Security.TlsSession
+    {
+        public TlsBufferSession() { }
+        public System.Net.Security.TlsOperationStatus Handshake(System.ReadOnlySpan<byte> source, System.Span<byte> destination, out int bytesConsumed, out int bytesWritten) { throw null; }
+        public System.Net.Security.TlsOperationStatus Write(System.ReadOnlySpan<byte> source, System.Span<byte> destination, out int bytesConsumed, out int bytesWritten) { throw null; }
+        public System.Net.Security.TlsOperationStatus Read(System.ReadOnlySpan<byte> source, System.Span<byte> destination, out int bytesConsumed, out int bytesWritten) { throw null; }
+        public System.Net.Security.TlsOperationStatus Shutdown(System.Span<byte> ciphertext, out int bytesWritten) { throw null; }
+        public System.Net.Security.TlsOperationStatus DrainPendingOutput(System.Span<byte> ciphertext, out int bytesWritten) { throw null; }
+        public System.Net.Security.TlsOperationStatus RequestClientCertificate(System.Span<byte> ciphertext, out int bytesWritten) { throw null; }
+    }
+    [System.Diagnostics.CodeAnalysis.ExperimentalAttribute("SYSLIB5007", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")]
+    public sealed partial class TlsSocketSession : System.Net.Security.TlsSession
+    {
+        public TlsSocketSession(System.Net.Sockets.SafeSocketHandle socket) { }
+        public System.Net.Sockets.SafeSocketHandle Socket { get { throw null; } }
+        public System.Net.Security.TlsOperationStatus Handshake() { throw null; }
+        public System.Net.Security.TlsOperationStatus Read(System.Span<byte> buffer, out int bytesRead) { throw null; }
+        public System.Net.Security.TlsOperationStatus Write(System.ReadOnlySpan<byte> buffer, out int bytesWritten) { throw null; }
+        public System.Net.Security.TlsOperationStatus Shutdown() { throw null; }
+        public System.Net.Security.TlsOperationStatus RequestClientCertificate() { throw null; }
+    }
 }
 namespace System.Security.Authentication
 {

src/libraries/System.Net.Security/ref/System.Net.Security.csproj

@@ -12,6 +12,7 @@
     <ProjectReference Include="$(LibrariesProjectRoot)System.Collections\ref\System.Collections.csproj" />
     <ProjectReference Include="$(LibrariesProjectRoot)System.Memory\ref\System.Memory.csproj" />
     <ProjectReference Include="$(LibrariesProjectRoot)System.Net.Primitives\ref\System.Net.Primitives.csproj" />
+    <ProjectReference Include="$(LibrariesProjectRoot)System.Net.Sockets\ref\System.Net.Sockets.csproj" />
     <ProjectReference Include="$(LibrariesProjectRoot)System.Runtime\ref\System.Runtime.csproj" />
     <ProjectReference Include="$(LibrariesProjectRoot)System.Security.Cryptography\ref\System.Security.Cryptography.csproj" />
   </ItemGroup>