From 54019a963922173efecb48d1aa5b49a63edceddd Mon Sep 17 00:00:00 2001 From: shaq918 Date: Tue, 7 Jul 2026 18:36:52 -0700 Subject: [PATCH] tls: prevent engine hang from spinning TLS read/write retry loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a TLS connection's underlying TCP socket continuously reports EPOLLIN but SSL_read() returns SSL_ERROR_WANT_READ (no complete TLS records available), flb_tls_net_read_async() enters a tight spin loop: 1. SSL_read() → read(fd) → EAGAIN 2. OpenSSL returns SSL_ERROR_WANT_READ → FLB_TLS_WANT_READ 3. io_tls_event_switch() is a no-op (event already registered for READ) 4. flb_coro_yield() yields, engine sees EPOLLIN, resumes immediately 5. goto retry_read → back to step 1 This monopolizes the main engine thread (flb-pipeline), starving all other event processing: output flushes never execute, scheduler timers don't fire, and new connections can't be accepted. The entire output pipeline hangs while the process appears healthy. Observed in production with Fluent Bit v3.2.2 where a dead forward input TLS connection (TCP ESTABLISHED, rx_queue=0, no data for 3.6 days) caused the engine thread to spin at ~8,500 failed read() calls/second. The output_proc_records_total counter froze, but output_errors_total remained 0. Same pattern confirmed on a second independent node with a different dead connection (47 days idle). Validated by killing the dead TCP connections on the live system: output_proc_records immediately jumped from 9,123,353 to 9,264,379 (+141,026 records flushed) and the engine thread resumed normal operation — proving the dead TLS connections were the sole cause. Fix: Add FLB_TLS_WANT_READ_MAX_RETRIES (800) to cap consecutive WANT_READ/WANT_WRITE iterations with no progress in both flb_tls_net_read_async() and flb_tls_net_write_async(). When the limit is exceeded, return -1 to close the stale connection and allow the engine thread to resume normal event processing. Signed-off-by: shaq918 Fixes: #9927 --- include/fluent-bit/tls/flb_tls.h | 7 +++ src/tls/flb_tls.c | 73 ++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/include/fluent-bit/tls/flb_tls.h b/include/fluent-bit/tls/flb_tls.h index 018231218e2..f3c1f485eee 100644 --- a/include/fluent-bit/tls/flb_tls.h +++ b/include/fluent-bit/tls/flb_tls.h @@ -35,6 +35,13 @@ #define FLB_TLS_WANT_READ -0x7e4 #define FLB_TLS_WANT_WRITE -0x7e6 +/* + * Maximum consecutive WANT_READ/WANT_WRITE retries before treating the + * connection as stale. Prevents a dead TLS connection from spinning the + * engine thread in a tight yield/resume loop (see #9927). + */ +#define FLB_TLS_WANT_READ_MAX_RETRIES 800 + /* Cert Flags */ #define FLB_TLS_CA_ROOT 1 #define FLB_TLS_CERT 2 diff --git a/src/tls/flb_tls.c b/src/tls/flb_tls.c index 40961b6a12a..0239b6475cf 100644 --- a/src/tls/flb_tls.c +++ b/src/tls/flb_tls.c @@ -17,6 +17,8 @@ * limitations under the License. */ +#include + #include #include #include @@ -393,17 +395,50 @@ int flb_tls_net_read_async(struct flb_coro *co, struct mk_event event_backup; struct flb_tls *tls; int ret; + int want_read_retries; tls = session->tls; event_restore_needed = FLB_FALSE; + /* + * Track consecutive SSL_ERROR_WANT_READ iterations with no progress. + * When a TLS connection's underlying socket continuously reports + * EPOLLIN but SSL_read() returns WANT_READ (no actual TLS data), + * the coroutine bounces between yield and resume in a tight loop, + * monopolizing the engine thread and starving all other processing + * (output flushes, timers, new connections). + * + * This happens when a dead TCP connection stays ESTABLISHED but + * has no TLS records to deliver — the kernel signals "readable" + * but OpenSSL needs more data. + * + * See: https://github.com/fluent/fluent-bit/issues/9927 + */ + want_read_retries = 0; + io_tls_backup_event(session->connection, &event_backup); retry_read: ret = tls->api->net_read(session, buf, len); if (ret == FLB_TLS_WANT_READ) { + want_read_retries++; + + if (want_read_retries > FLB_TLS_WANT_READ_MAX_RETRIES) { + flb_warn("[tls] connection #%i exceeded maximum read retries (%i), " + "closing stale connection", + session->connection->fd, + FLB_TLS_WANT_READ_MAX_RETRIES); + + session->connection->coroutine = NULL; + session->connection->net_error = ETIMEDOUT; + + io_tls_restore_event(session->connection, &event_backup); + + return -1; + } + event_restore_needed = FLB_TRUE; session->connection->coroutine = co; @@ -414,6 +449,8 @@ int flb_tls_net_read_async(struct flb_coro *co, goto retry_read; } else if (ret == FLB_TLS_WANT_WRITE) { + /* Progress on a different axis, reset the read retry counter */ + want_read_retries = 0; event_restore_needed = FLB_TRUE; session->connection->coroutine = co; @@ -500,11 +537,13 @@ int flb_tls_net_write_async(struct flb_coro *co, size_t total; int ret; struct flb_tls *tls; + int want_retries; total = 0; tls = session->tls; event_restore_needed = FLB_FALSE; + want_retries = 0; io_tls_backup_event(session->connection, &event_backup); @@ -516,6 +555,23 @@ int flb_tls_net_write_async(struct flb_coro *co, len - total); if (ret == FLB_TLS_WANT_WRITE) { + want_retries++; + + if (want_retries > FLB_TLS_WANT_READ_MAX_RETRIES) { + flb_warn("[tls] connection #%i exceeded maximum write retries (%i), " + "closing stale connection", + session->connection->fd, + FLB_TLS_WANT_READ_MAX_RETRIES); + + session->connection->coroutine = NULL; + session->connection->net_error = ETIMEDOUT; + *out_len = total; + + io_tls_restore_event(session->connection, &event_backup); + + return -1; + } + event_restore_needed = FLB_TRUE; io_tls_event_switch(session, MK_EVENT_WRITE); @@ -525,6 +581,23 @@ int flb_tls_net_write_async(struct flb_coro *co, goto retry_write; } else if (ret == FLB_TLS_WANT_READ) { + want_retries++; + + if (want_retries > FLB_TLS_WANT_READ_MAX_RETRIES) { + flb_warn("[tls] connection #%i exceeded maximum write retries (%i), " + "closing stale connection", + session->connection->fd, + FLB_TLS_WANT_READ_MAX_RETRIES); + + session->connection->coroutine = NULL; + session->connection->net_error = ETIMEDOUT; + *out_len = total; + + io_tls_restore_event(session->connection, &event_backup); + + return -1; + } + event_restore_needed = FLB_TRUE; io_tls_event_switch(session, MK_EVENT_READ);