From b7827ed46eb590b96f27b48c891b9d39b8aef702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Thu, 9 Jul 2026 11:59:44 +0200 Subject: [PATCH 01/29] operator: add Broker CRD API, conditions, and roll-grant feature flags Introduces the cluster.redpanda.com/v1alpha2 Broker custom resource: a per-broker unit of pod + storage management with an explicit decommission intent field, a persisted networkIndex, and pool-aware pod naming (- for the default pool, -- for named pools). PodOutdated() defines the rotation contract: a pod drifts when its config checksum or restart-requiring cluster-config version annotation no longer matches the desired pod template. Adds the Broker status conditions (Ready, PodScheduled, StorageBound, BrokerRegistered, ConfigOutdated, Quiesced, Stable) to statuses.yaml and the roll-grant lease primitives to the feature package (RFC Q4): the grant is / where the checksum doubles as a generation so stale grants are rejected, and the TTL is a safety valve against controller restarts, not a pacing mechanism. The broker-deletion-policy annotation (cascade|orphan) declares what happens to pods and PVCs on cluster teardown. The Broker CRD registers under the experimental group (opt-in via `operator crd --experimental`). Co-Authored-By: Claude Fable 5 --- go.work.sum | 36 +++++ .../api/redpanda/v1alpha2/broker_helpers.go | 63 +++++++++ .../redpanda/v1alpha2/broker_helpers_test.go | 63 +++++++++ .../api/redpanda/v1alpha2/broker_types.go | 35 ++++- .../redpanda/v1alpha2/testdata/crd-docs.adoc | 2 +- .../bases/cluster.redpanda.com_brokers.yaml | 5 + .../internal/statuses/zz_generated_status.go | 129 ++++++++++++++++-- .../statuses/zz_generated_status_test.go | 75 ++++++++++ operator/pkg/feature/flags.go | 64 +++++++++ operator/pkg/feature/flags_test.go | 56 ++++++++ operator/statuses.yaml | 42 +++++- 11 files changed, 548 insertions(+), 22 deletions(-) create mode 100644 operator/api/redpanda/v1alpha2/broker_helpers.go create mode 100644 operator/api/redpanda/v1alpha2/broker_helpers_test.go create mode 100644 operator/pkg/feature/flags_test.go diff --git a/go.work.sum b/go.work.sum index ea0eefbd6e..29d142f1d5 100644 --- a/go.work.sum +++ b/go.work.sum @@ -112,6 +112,8 @@ cloud.google.com/go/aiplatform v1.108.0 h1:5E+YsR0Zi6ZJiC7yohfK13Ufoc29NoApRBajK cloud.google.com/go/aiplatform v1.108.0/go.mod h1:4rwKOMdubQOND81AlO3EckcskvEFCYSzXKfn42GMm8k= cloud.google.com/go/aiplatform v1.116.0 h1:Qc8tv4DD6IbQfDKDd1Hu2qeGeYxTKTeZ7GH0vQrLAm8= cloud.google.com/go/aiplatform v1.116.0/go.mod h1:AdvoUUSXh9ykwEazibd3Fj6OUGrIiZwvZrvm4j5OdkU= +cloud.google.com/go/aiplatform v1.120.0 h1:jKWTpEs+xoUhDa1FMdSuhMcEQYyUiMdufGyX3zvtLVQ= +cloud.google.com/go/aiplatform v1.120.0/go.mod h1:6mDthfmy0oS1EQhVFdijoxkVdI2+HIZkpuGTBpedeCg= cloud.google.com/go/analytics v0.25.1/go.mod h1:hrAWcN/7tqyYwF/f60Nph1yz5UE3/PxOPzzFsJgtU+Y= cloud.google.com/go/analytics v0.27.1 h1:Tv2+kl/8xLtsoO/wCmiQTWzrnRiXHF0F0F4mUs9pUhA= cloud.google.com/go/analytics v0.27.1/go.mod h1:2itQDvSWyGiBvs80ocjFjfu/ZUIo25fC93hsEX4fnoU= @@ -155,6 +157,8 @@ cloud.google.com/go/area120 v0.9.6 h1:iJrZ6AleZr4l+q0/fWVANFOhs90KiSB1Ccait5OYyN cloud.google.com/go/area120 v0.9.6/go.mod h1:qKSokqe0iTmwBDA3tbLWonMEnh0pMAH4YxiceiHUed4= cloud.google.com/go/area120 v0.9.7 h1:BbpzLwaIXVPorrrzTH+ni7P5mLemmPPfSZ7o39k7zQc= cloud.google.com/go/area120 v0.9.7/go.mod h1:5nJ0yksmjOMfc4Zpk+okWfJ3A1004FvB82rfia+ZLaY= +cloud.google.com/go/area120 v0.10.0 h1:8oNFb5jJZLh0/prs0yJiYJpIC5qDcQ8u+Mfhe30pSx0= +cloud.google.com/go/area120 v0.10.0/go.mod h1:Xg3fKl4xU3UVai9wsI1FXwNU8wSCDYT7dFZfwJKViAM= cloud.google.com/go/artifactregistry v1.15.1/go.mod h1:ExJb4VN+IMTQWO5iY+mjcY19Rz9jUxCVGZ1YuyAgPBw= cloud.google.com/go/artifactregistry v1.16.3 h1:FPTr8KmB+K29uOg+3IITqdXHd1sP+Y+hxQtMvxW0MfM= cloud.google.com/go/artifactregistry v1.16.3/go.mod h1:eiLO70Qh5Z9Jbwctl0KdW5VzJ5HncWgNaYN0NdF8lmM= @@ -171,6 +175,8 @@ cloud.google.com/go/asset v1.21.1 h1:i55wWC/EwVdHMyJgRfbLp/L6ez4nQuOpZwSxkuqN9ek cloud.google.com/go/asset v1.21.1/go.mod h1:7AzY1GCC+s1O73yzLM1IpHFLHz3ws2OigmCpOQHwebk= cloud.google.com/go/asset v1.22.0 h1:81Ru5hjHfiGtk+u/Ix69eaWieKpvm7Ce7UHtcZhOLbk= cloud.google.com/go/asset v1.22.0/go.mod h1:q80JP2TeWWzMCazYnrAfDf36aQKf1QiKzzpNLflJwf8= +cloud.google.com/go/asset v1.22.1 h1:wimPPWu5gjBkPY1576vr+YxfoLKVhAK9zM2XrEpdKQ4= +cloud.google.com/go/asset v1.22.1/go.mod h1:NlvWwmca7CX6BIBEdRNxOocH6DowmBghAAHucOHuHng= cloud.google.com/go/assuredworkloads v1.12.1/go.mod h1:nBnkK2GZNSdtjU3ER75oC5fikub5/+QchbolKgnMI/I= cloud.google.com/go/assuredworkloads v1.12.5 h1:Ul3bAchXqIYtG7eTJHuzDOPd+NXlXL/VQGLra5sW3MU= cloud.google.com/go/assuredworkloads v1.12.5/go.mod h1:OHjBWxs611PdU/VkGDoNQ/SFZHIYQTPtZlfDAUWN8K0= @@ -233,6 +239,8 @@ cloud.google.com/go/bigquery v1.72.0 h1:D/yLju+3Ens2IXx7ou1DJ62juBm+/coBInn4VVOg cloud.google.com/go/bigquery v1.72.0/go.mod h1:GUbRtmeCckOE85endLherHD9RsujY+gS7i++c1CqssQ= cloud.google.com/go/bigquery v1.73.1 h1:v//GZwdhtmCbZ87rOnxz7pectOGFS1GNRvrGTvLzka4= cloud.google.com/go/bigquery v1.73.1/go.mod h1:KSLx1mKP/yGiA8U+ohSrqZM1WknUnjZAxHAQZ51/b1k= +cloud.google.com/go/bigquery v1.74.0 h1:Q6bAMv+eyvufOpIrfrYxhM46qq1D3ZQTdgUDQqKS+n8= +cloud.google.com/go/bigquery v1.74.0/go.mod h1:iViO7Cx3A/cRKcHNRsHB3yqGAMInFBswrE9Pxazsc90= cloud.google.com/go/bigtable v1.33.0/go.mod h1:HtpnH4g25VT1pejHRtInlFPnN5sjTxbQlsYBjh9t5l0= cloud.google.com/go/bigtable v1.36.0 h1:GU4XWYb7H9XYHvksDA/jixUZUv2ZESNS48QEc/qduV4= cloud.google.com/go/bigtable v1.36.0/go.mod h1:u98oqNAXiAufepkRGAd95lq2ap4kHGr3wLeFojvJwew= @@ -356,6 +364,8 @@ cloud.google.com/go/dataform v0.12.0 h1:0eCPTPUC/RZ863aVfXTJLkg0tEpdpn62VD6ywSmm cloud.google.com/go/dataform v0.12.0/go.mod h1:PuDIEY0lSVuPrZqcFji1fmr5RRvz3DGz4YP/cONc8g4= cloud.google.com/go/dataform v0.12.1 h1:yf6Up6m1FUt+YB5CBgNtIZfz2OzjuNBdzZWV3SLSVNE= cloud.google.com/go/dataform v0.12.1/go.mod h1:atGS8ReRjfNDUQib0X/o/7Gi2bqHI2G7/J86LKiGimE= +cloud.google.com/go/dataform v0.13.0 h1:z4nzTOqGSkJ5ePyJLQiUDTBsPHdokzvNNDhGebGQEUM= +cloud.google.com/go/dataform v0.13.0/go.mod h1:U3fqrPY5jAcFh1a8rQb4a+PQ7zKlc5qfgotFZ+luKPo= cloud.google.com/go/datafusion v1.8.1/go.mod h1:I5+nRt6Lob4g1eCbcxP4ayRNx8hyOZ8kA3PB/vGd9Lo= cloud.google.com/go/datafusion v1.8.5 h1:e4yglwl3yE/IN2/xfRdYckThuVCAlP22+srHaJ0DY0Y= cloud.google.com/go/datafusion v1.8.5/go.mod h1:xMoW16ciCOQpS8rNUDU1tWgHkhbQ3KKaV9o7UTggEtQ= @@ -387,6 +397,8 @@ cloud.google.com/go/dataproc/v2 v2.11.2 h1:KhC8wdLILpAs17yeTG6Miwg1v0nOP/OXD+9QN cloud.google.com/go/dataproc/v2 v2.11.2/go.mod h1:xwukBjtfiO4vMEa1VdqyFLqJmcv7t3lo+PbLDcTEw+g= cloud.google.com/go/dataproc/v2 v2.15.0 h1:I/Yux/d8uaxf3W+d59kolGTOc52+VZaL6RzJw7oDOeg= cloud.google.com/go/dataproc/v2 v2.15.0/go.mod h1:tSdkodShfzrrUNPDVEL6MdH9/mIEvp/Z9s9PBdbsZg8= +cloud.google.com/go/dataproc/v2 v2.16.0 h1:0g2hnjlQ8SQTnNeu+Bqqa61QPssfSZF3t+9ldRmx+VQ= +cloud.google.com/go/dataproc/v2 v2.16.0/go.mod h1:HlzFg8k1SK+bJN3Zsy2z5g6OZS1D4DYiDUgJtF0gJnE= cloud.google.com/go/dataqna v0.9.1/go.mod h1:86DNLE33yEfNDp5F2nrITsmTYubMbsF7zQRzC3CcZrY= cloud.google.com/go/dataqna v0.9.5 h1:C3bXU5pqhFahiuBhHdbLGf7WDnm0qZxLV4e5HWj41Qo= cloud.google.com/go/dataqna v0.9.5/go.mod h1:UFRToVzSTCgwDkeSa4J0WE6bmbemdOZhUCUfs+DlJFc= @@ -426,6 +438,8 @@ cloud.google.com/go/dialogflow v1.70.0 h1:76UcutgQuV8s1t2yIz6rdUCkosDT6nT3LCsybP cloud.google.com/go/dialogflow v1.70.0/go.mod h1:mP4XrpgDvPYBP+cdLxFC1WJJlkwuy0H8L1Lada9No/M= cloud.google.com/go/dialogflow v1.75.0 h1:4TFe375Ti6HjUgJd4JWFEqyCAMhRZFLJUIeMF16TqTo= cloud.google.com/go/dialogflow v1.75.0/go.mod h1:z1W1ZogmigYVtP5YmyeUh+D219VCjdd3VJqY76PG3gA= +cloud.google.com/go/dialogflow v1.76.0 h1:hP9GY9TSVlZ277IGCPQjem9RW1PDtfYJw98DkRGt+t4= +cloud.google.com/go/dialogflow v1.76.0/go.mod h1:mdLkMmSCghfcP85X9dFBlirC1OssS65KE5hrrSz2GXY= cloud.google.com/go/dlp v1.19.0/go.mod h1:cr8dKBq8un5LALiyGkz4ozcwzt3FyTlOwA4/fFzJ64c= cloud.google.com/go/dlp v1.22.0 h1:mNClUVvLdfqBhSgLm3BLuchJ6xowqNVn1K3Why9y5xU= cloud.google.com/go/dlp v1.22.0/go.mod h1:2cMTKdeReZI64BDsYzsBZFtXdDqb3nhDKHRsRUl7J9Y= @@ -444,6 +458,8 @@ cloud.google.com/go/documentai v1.39.0 h1:ngPNmVUb3xxj6/j7ruK4vmOMm+XTJO0Vi7Jkm0 cloud.google.com/go/documentai v1.39.0/go.mod h1:KmlLO93F7GRU8dENXRxvt+7V8o7eCG6Y6WDitKbcYJs= cloud.google.com/go/documentai v1.41.0 h1:cAhuqgmk0KkhQmlpWp+/Vo/V9sTaFarKof663urdGps= cloud.google.com/go/documentai v1.41.0/go.mod h1:AT+3TV4vXGT06eyNmVmyivzN/dlcVOXlh6ufl1X9rAI= +cloud.google.com/go/documentai v1.42.0 h1:FErf7mEjf3TBGiwcXQCsLrQ3mUqryTKa09NiO1L8Y90= +cloud.google.com/go/documentai v1.42.0/go.mod h1:CABOUzRNOuvb/QwJS2LS80Hpqbu3UW2afyRKTYuW7bo= cloud.google.com/go/domains v0.10.1/go.mod h1:RjDl3K8iq/ZZHMVqfZzRuBUr5t85gqA6LEXQBeBL5F4= cloud.google.com/go/domains v0.10.5 h1:6JmxjzqJ5HSYWVK5EWJA73H4JJ2K+1M2mLuxnT1Wn1w= cloud.google.com/go/domains v0.10.5/go.mod h1:VP7djhZJy47uxUoJGfDilXpUnAaIExcHL86vv3yfaQs= @@ -579,6 +595,8 @@ cloud.google.com/go/kms v1.23.2 h1:4IYDQL5hG4L+HzJBhzejUySoUOheh3Lk5YT4PCyyW6k= cloud.google.com/go/kms v1.23.2/go.mod h1:rZ5kK0I7Kn9W4erhYVoIRPtpizjunlrfU4fUkumUp8g= cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ= cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk= +cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU= +cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58= cloud.google.com/go/language v1.14.1/go.mod h1:WaAL5ZdLLBjiorXl/8vqgb6/Fyt2qijl96c1ZP/vdc8= cloud.google.com/go/language v1.14.4 h1:rQwSEiY2uPD11xelCr8QXidWIV/MzF8B9QXAUdfQph8= cloud.google.com/go/language v1.14.4/go.mod h1:EqwoMieV6UsNeqHV2tRxuhmfDyC3YqEu1er53CrRkeA= @@ -631,6 +649,8 @@ cloud.google.com/go/maps v1.25.0 h1:E4KozZ8UzxTUA3R/QtZg1vw3f65miBUnoUHkSwf03dA= cloud.google.com/go/maps v1.25.0/go.mod h1:+auempdONAP8emtm48aCfNo1ZC+3CJniRA1h8J4u7bY= cloud.google.com/go/maps v1.27.0 h1:ttCD1VzSg8BhnkgPkYygYVUHGyK/yHwEz5qJVps+0pM= cloud.google.com/go/maps v1.27.0/go.mod h1:6EWjz3AFh52w3qe2reWShQDmGRtryhP7NAfGolnr9+g= +cloud.google.com/go/maps v1.29.0 h1:iAlFpnckCAshFpmHPDUYpasXn0pH4OVMDfkb3jB/fDQ= +cloud.google.com/go/maps v1.29.0/go.mod h1:FNATcM5ziB2TDE2IVWH4f/yeXc+SbUk1X+bmKjR8HEA= cloud.google.com/go/mediatranslation v0.9.1/go.mod h1:vQH1amULNhSGryBjbjLb37g54rxrOwVxywS8WvUCsIU= cloud.google.com/go/mediatranslation v0.9.5 h1:r62i+EKWNI7wvhcOXuVKCp8skeeRzWd0pO+8qtqTN1c= cloud.google.com/go/mediatranslation v0.9.5/go.mod h1:JGsL9cldTUtRi3u6Q+BMXzY1zZFOWdbmZLf1C69G2Zs= @@ -672,6 +692,8 @@ cloud.google.com/go/networkconnectivity v1.19.1 h1:n0IzhdgSNzIKQygWwDV8yKRXkZpX3 cloud.google.com/go/networkconnectivity v1.19.1/go.mod h1:Q5v6uNNNz8BP232uuXM66XgWML9m379xhwv58Y+8Kb0= cloud.google.com/go/networkconnectivity v1.20.0 h1:A0uRcZJdq7F6LYWcc2NIea3h0i7p6kY1/CyLavOTG0I= cloud.google.com/go/networkconnectivity v1.20.0/go.mod h1:9MzGwD4ljiq+Z2Pg3ue27OEewCuHz7IUfw1fITrIdSw= +cloud.google.com/go/networkconnectivity v1.21.0 h1:WS5XTNWyLODLO5YmftQmDIZtAa2DYYmRf/neRCLIWHA= +cloud.google.com/go/networkconnectivity v1.21.0/go.mod h1:XC1UJ+tqBsLWz73dqrMc7kUvdTv0FIxtDGv6YntTBO0= cloud.google.com/go/networkmanagement v1.14.1/go.mod h1:3Ds8FZ3ZHjTVEedsBoZi9ef9haTE14iS6swTSqM39SI= cloud.google.com/go/networkmanagement v1.18.2 h1:6yLMeEOVssCNdOutIlBplASms6LqR8rTDAWne9TEPB4= cloud.google.com/go/networkmanagement v1.18.2/go.mod h1:QOOTm+LgXEPeA9u9bAeDETBYkibzMVTYH4mIi9GJATc= @@ -681,6 +703,8 @@ cloud.google.com/go/networkmanagement v1.20.1 h1:W5zdnH332yJFADTXlsHWLVkiXLKGWHj cloud.google.com/go/networkmanagement v1.20.1/go.mod h1:clG/5Yt0wQ57qSH6Yh7oehQYlobHw3F6nb3Pn4ig5hU= cloud.google.com/go/networkmanagement v1.22.0 h1:ibvtNUXQj07zlSbfwmS11YWan4+W8tQ/2L7zXOZ4V7U= cloud.google.com/go/networkmanagement v1.22.0/go.mod h1:RGR62aLOlm72C7DT/3yaMUK43oill6hj9wqktUQ8h6Q= +cloud.google.com/go/networkmanagement v1.23.0 h1:PiteUY9H2u+wMgT1dQjD93PKI90o10RgL8+OhUov970= +cloud.google.com/go/networkmanagement v1.23.0/go.mod h1:QTYCWp5UxUnU280SqF7AX/mf6NhsqKblmLeCALQmx5c= cloud.google.com/go/networksecurity v0.10.1/go.mod h1:tatO1hYJ9nNChLHOFdsjex5FeqZBlPQgKdKOex7REpU= cloud.google.com/go/networksecurity v0.10.5 h1:42ODFhAJ+haEWqv5N5MH171FIGDXMu6cNIbzuOT0zOs= cloud.google.com/go/networksecurity v0.10.5/go.mod h1:CqJMtLG67gxHEAjGjccwEm5a7Tb6h0kPtHK5SEHnwMc= @@ -885,6 +909,8 @@ cloud.google.com/go/speech v1.28.1 h1:L8kq/CypGn6y/FbipyAPyn1L9JDW4CK3zkcAe4oDN7 cloud.google.com/go/speech v1.28.1/go.mod h1:+EN8Zuy6y2BKe9P1RAmMaFPAgBns6m+XMgXAfkYtSSE= cloud.google.com/go/speech v1.29.0 h1:ehOzN/IsAhjjAtWg4fI8A3iNtonb1N8yWjofVhSTv+c= cloud.google.com/go/speech v1.29.0/go.mod h1:wtUmIS/h0ZYU6cPA9klcyST3f6i2FdnvNDqENjrRDds= +cloud.google.com/go/speech v1.30.0 h1:R+KGIbRMrj8jA4U6Qea8hqCMsAEdg576ShNsmRr4gcQ= +cloud.google.com/go/speech v1.30.0/go.mod h1:F2+NJujR8uzDLd6bwy5kgtVycxvEq06nzvzz5eQ/gMo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -1816,6 +1842,8 @@ github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UN github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= @@ -1826,6 +1854,8 @@ github.com/go-git/go-git/v5 v5.16.3 h1:Z8BtvxZ09bYm/yYNgPKCzgWtaRqDTgIKRgIRHBfU6 github.com/go-git/go-git/v5 v5.16.3/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= github.com/go-git/go-git/v5 v5.17.0 h1:AbyI4xf+7DsjINHMu35quAh4wJygKBKBuXVjV/pxesM= github.com/go-git/go-git/v5 v5.17.0/go.mod h1:f82C4YiLx+Lhi8eHxltLeGC5uBTXSFa6PC5WW9o4SjI= +github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00= +github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -2650,6 +2680,8 @@ github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0= github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -2953,6 +2985,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/twmb/avro v1.3.0 h1:VFDYzwwe5J3ojl0MmMSbxhjRXX2Wt5NOqWZiQPunpBc= github.com/twmb/avro v1.3.0/go.mod h1:TUQS96Ptl8tDRyK0Jw91FXIVCoPKz4sXxvUSShiG5FA= +github.com/twmb/avro v1.4.1 h1:pzXZivLS0YkI7lntpxnu6aYkNJBUJ3vSp6uzEGLeQOI= +github.com/twmb/avro v1.4.1/go.mod h1:X0fT1dY2xcbV4YuCE4mYro+qljHl4kUF5uA/2z1rgSk= github.com/twmb/franz-go v1.16.1/go.mod h1:/pER254UPPGp/4WfGqRi+SIRGE50RSQzVubQp6+N4FA= github.com/twmb/franz-go v1.18.0/go.mod h1:zXCGy74M0p5FbXsLeASdyvfLFsBvTubVqctIaa5wQ+I= github.com/twmb/franz-go v1.18.1/go.mod h1:Uzo77TarcLTUZeLuGq+9lNpSkfZI+JErv7YJhlDjs9M= @@ -4012,6 +4046,8 @@ google.golang.org/genproto/googleapis/bytestream v0.0.0-20250929231259-57b25ae83 google.golang.org/genproto/googleapis/bytestream v0.0.0-20250929231259-57b25ae835d4/go.mod h1:YUQUKndxDbAanQC0ln4pZ3Sis3N5sqgDte2XQqufkJc= google.golang.org/genproto/googleapis/bytestream v0.0.0-20260202165425-ce8ad4cf556b h1:9aVE1sMjpFNretuehjEjpYcrdIkGdmT8NWkWNIqlasE= google.golang.org/genproto/googleapis/bytestream v0.0.0-20260202165425-ce8ad4cf556b/go.mod h1:Tej9lWiwVvQJP+b43pjJIsr/3mZycXWCIyoiXmbFf40= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20260226221140-a57be14db171 h1:zeSmMyGeCh9H7xJRBK+IS3b272WQpcvXUnLPhNuYLWI= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20260226221140-a57be14db171/go.mod h1:9amqk/8LQWEC4RjyUxMx1DebyQ7hZB9gvl67bHmgZ2E= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= diff --git a/operator/api/redpanda/v1alpha2/broker_helpers.go b/operator/api/redpanda/v1alpha2/broker_helpers.go new file mode 100644 index 0000000000..7f4b85e526 --- /dev/null +++ b/operator/api/redpanda/v1alpha2/broker_helpers.go @@ -0,0 +1,63 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package v1alpha2 + +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +// PodName returns the name of the pod this Broker manages. Pods keep their +// StatefulSet ordinal names: `-` for the default node pool +// and `--` for pools declared in the cluster's +// nodePools — matching the per-pool StatefulSet naming. The owning pool is +// read from the Broker's node pool label. +func (b *Broker) PodName() string { + if b.Spec.ClusterRef.IsNodePool() { + // for nodepools, ClusterRef.Name is a NodePool name, + // so we need to rely on the cluster name in the label + return fmt.Sprintf("%s-%s-%d", b.Labels[ClusterNameLabel], b.Spec.ClusterRef.Name, ptr.Deref(b.Spec.NetworkIndex, 0)) + } + name := b.Spec.ClusterRef.Name + if pool := b.Labels[NodePoolLabel]; pool != "" && !strings.EqualFold(pool, DefaultNodePoolName) { + name = fmt.Sprintf("%s-%s", name, pool) + } + return fmt.Sprintf("%s-%d", name, ptr.Deref(b.Spec.NetworkIndex, 0)) +} + +// PodOutdated reports whether the live pod has drifted from the Broker's +// desired pod template on the keys that demand a rotation: the config +// checksum and the restart-requiring cluster-config version. Pods inherit +// both at creation, so a recreated pod never reports drift. +func (b *Broker) PodOutdated(pod *corev1.Pod) bool { + for _, key := range []string{BrokerConfigChecksumAnnotation, BrokerClusterConfigVersionAnnotation} { + if desired := b.Spec.PodTemplate.Annotations[key]; desired != "" && pod.Annotations[key] != desired { + return true + } + } + return false +} + +func (b *Broker) BuildPod(podName string) *corev1.Pod { + pod := &corev1.Pod{ + Spec: b.Spec.PodTemplate.Spec, + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: b.Namespace, + Annotations: b.Spec.PodTemplate.Annotations, + Labels: b.Spec.PodTemplate.Labels, + }, + } + return pod +} diff --git a/operator/api/redpanda/v1alpha2/broker_helpers_test.go b/operator/api/redpanda/v1alpha2/broker_helpers_test.go new file mode 100644 index 0000000000..9838468fc0 --- /dev/null +++ b/operator/api/redpanda/v1alpha2/broker_helpers_test.go @@ -0,0 +1,63 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package v1alpha2 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +func TestBrokerPodName(t *testing.T) { + v1Broker := func(pool string, index int32) *Broker { + b := &Broker{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns"}, + Spec: BrokerSpec{ + ClusterRef: ClusterRef{ + Group: ptr.To("redpanda.vectorized.io"), + Kind: ptr.To("Cluster"), + Name: "rp", + }, + NetworkIndex: ptr.To(index), + }, + } + if pool != "" { + b.Labels = map[string]string{NodePoolLabel: pool} + } + return b + } + + // Default pool (explicit or unlabeled): pods carry the bare cluster name. + assert.Equal(t, "rp-0", v1Broker("default", 0).PodName()) + assert.Equal(t, "rp-2", v1Broker("", 2).PodName()) + // Pools declared in Cluster.spec.nodePools: --, + // matching the per-pool StatefulSet naming. + assert.Equal(t, "rp-high-mem-1", v1Broker("high-mem", 1).PodName()) + + // NodePool ref: ClusterRef names the pool, the cluster half comes from + // ClusterNameLabel — pods are --. + nodePoolBroker := &Broker{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Labels: map[string]string{ClusterNameLabel: "rp"}, + }, + Spec: BrokerSpec{ + ClusterRef: ClusterRef{ + Group: ptr.To("cluster.redpanda.com"), + Kind: ptr.To("NodePool"), + Name: "high-mem", + }, + NetworkIndex: ptr.To(int32(1)), + }, + } + assert.Equal(t, "rp-high-mem-1", nodePoolBroker.PodName()) +} diff --git a/operator/api/redpanda/v1alpha2/broker_types.go b/operator/api/redpanda/v1alpha2/broker_types.go index cac87ba2ba..2f7d8efcc9 100644 --- a/operator/api/redpanda/v1alpha2/broker_types.go +++ b/operator/api/redpanda/v1alpha2/broker_types.go @@ -29,6 +29,39 @@ const ( BrokerPhaseStuck BrokerPhase = "Stuck" ) +// BrokerConfigChecksumAnnotation carries the config checksum on both the +// Broker's desired pod template annotations and the live pod's annotations. +// A mismatch between the two marks the pod as needing rotation. +const BrokerConfigChecksumAnnotation = "config.redpanda.com/checksum" + +// BrokerClusterConfigVersionAnnotation carries the cluster-config version +// that last required a restart, stamped into the Broker's desired pod +// template when a restart-requiring central-config change lands (the +// `kubectl rollout restart` pattern: pods inherit it at creation, so a pod +// whose value differs from the desired template needs a rotation). The +// cluster controller only ever writes it to the Broker SPEC — never to pods. +const BrokerClusterConfigVersionAnnotation = "operator.redpanda.com/cluster-config-version" + +const ( + // NodePoolLabel carries the name of the node pool a Broker belongs to. + // It mirrors labels.NodePoolKey, redeclared here to keep the API package + // self-contained. + NodePoolLabel = "cluster.redpanda.com/nodepool" + + // ClusterNameLabel carries the name of the Redpanda cluster a Broker + // belongs to. It is required on Brokers whose ClusterRef points at a + // NodePool: the ref names the pool, not the cluster, and pod names are + // `--` — the cluster half is only recoverable + // from this label without fetching the NodePool. Whoever creates Broker + // CRs must always set it. + ClusterNameLabel = "cluster.redpanda.com/cluster-name" + + // DefaultNodePoolName mirrors vectorizedv1alpha1.DefaultNodePoolName: + // the implicit pool defined by a V1 Cluster's top-level replicas, whose + // StatefulSet (and pods) carry the bare cluster name. + DefaultNodePoolName = "default" +) + // +genclient // +kubebuilder:object:root=true // +kubebuilder:subresource:status @@ -43,7 +76,7 @@ type Broker struct { metav1.ObjectMeta `json:"metadata,omitempty"` Spec BrokerSpec `json:"spec,omitempty"` - // +kubebuilder:default={conditions: {{type: "Ready", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "PodScheduled", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "StorageBound", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "BrokerRegistered", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "Quiesced", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "Stable", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} + // +kubebuilder:default={conditions: {{type: "Ready", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "PodScheduled", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "StorageBound", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "BrokerRegistered", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "ConfigSynced", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "Quiesced", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}, {type: "Stable", status: "Unknown", reason: "NotReconciled", message: "Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} Status BrokerStatus `json:"status,omitempty"` } diff --git a/operator/api/redpanda/v1alpha2/testdata/crd-docs.adoc b/operator/api/redpanda/v1alpha2/testdata/crd-docs.adoc index f85e0b0204..4294a18ee0 100644 --- a/operator/api/redpanda/v1alpha2/testdata/crd-docs.adoc +++ b/operator/api/redpanda/v1alpha2/testdata/crd-docs.adoc @@ -478,7 +478,7 @@ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api- | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`. | | | *`spec`* __xref:{anchor_prefix}-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-brokerspec[$$BrokerSpec$$]__ | | | -| *`status`* __xref:{anchor_prefix}-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-brokerstatus[$$BrokerStatus$$]__ | | { conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:Ready] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:PodScheduled] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:StorageBound] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:BrokerRegistered] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:Quiesced] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:Stable]] } | +| *`status`* __xref:{anchor_prefix}-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-brokerstatus[$$BrokerStatus$$]__ | | { conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:Ready] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:PodScheduled] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:StorageBound] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:BrokerRegistered] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:ConfigSynced] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:Quiesced] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:NotReconciled status:Unknown type:Stable]] } | |=== diff --git a/operator/config/crd/bases/cluster.redpanda.com_brokers.yaml b/operator/config/crd/bases/cluster.redpanda.com_brokers.yaml index 28c4a01262..6e2bd9c2dc 100644 --- a/operator/config/crd/bases/cluster.redpanda.com_brokers.yaml +++ b/operator/config/crd/bases/cluster.redpanda.com_brokers.yaml @@ -8846,6 +8846,11 @@ spec: reason: NotReconciled status: Unknown type: BrokerRegistered + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: NotReconciled + status: Unknown + type: ConfigSynced - lastTransitionTime: "1970-01-01T00:00:00Z" message: Waiting for controller reason: NotReconciled diff --git a/operator/internal/statuses/zz_generated_status.go b/operator/internal/statuses/zz_generated_status.go index 2bae8bc89f..7d1844aed6 100644 --- a/operator/internal/statuses/zz_generated_status.go +++ b/operator/internal/statuses/zz_generated_status.go @@ -237,6 +237,15 @@ type BrokerStorageBoundCondition string // must be set by a controller when it subsequently reconciles a broker. type BrokerBrokerRegisteredCondition string +// BrokerConfigSyncedCondition - This condition indicates whether the broker's +// pod matches the desired pod template: both the config checksum and the +// restart-requiring cluster-config version. False means a rotation is pending +// (the pod will be recreated once the broker holds a valid roll-grant). +// +// This condition defaults to "Unknown" with a reason of "NotReconciled" and +// must be set by a controller when it subsequently reconciles a broker. +type BrokerConfigSyncedCondition string + // BrokerQuiescedCondition - This condition indicates the broker is no longer // reconciling for its current generation. // @@ -246,10 +255,10 @@ type BrokerQuiescedCondition string // BrokerStableCondition - This condition is a roll-up status for automation // (e.g. parent controller checking all brokers are stable before proceeding). -// It is True when Ready, StorageBound, BrokerRegistered, and Quiesced all -// evaluate to True. Each tracks one dimension: Ready = pod health, StorageBound -// = PVC binding, BrokerRegistered = cluster membership, Quiesced = -// reconciliation complete. +// It is True when Ready, StorageBound, BrokerRegistered, ConfigSynced, and +// Quiesced all evaluate to True. Each tracks one dimension: Ready = pod health, +// StorageBound = PVC binding, BrokerRegistered = cluster membership, +// ConfigSynced = no rotation pending, Quiesced = reconciliation complete. // // This condition defaults to "False" with a reason of "NotReconciled" and must // be set by a controller when it subsequently reconciles a broker. @@ -858,6 +867,10 @@ const ( // "PodScheduled" condition when it evaluates to False because the broker's pod // cannot be scheduled (e.g. node affinity, insufficient resources). BrokerPodScheduledReasonUnschedulable BrokerPodScheduledCondition = "Unschedulable" + // BrokerPodScheduledReasonPodMissing - This reason is used with the + // "PodScheduled" condition when it evaluates to False because the broker's pod + // does not exist (initial bootstrap, mid-rotation, or after decommission). + BrokerPodScheduledReasonPodMissing BrokerPodScheduledCondition = "PodMissing" // BrokerPodScheduledReasonError - This reason is used when a broker has only // been partially reconciled and we have early returned due to a retryable error // occurring prior to applying the desired broker state. @@ -906,6 +919,12 @@ const ( // "BrokerRegistered" condition when it evaluates to False because the broker // has not yet registered with the cluster. BrokerBrokerRegisteredReasonNotRegistered BrokerBrokerRegisteredCondition = "NotRegistered" + // BrokerBrokerRegisteredReasonIdentityChanged - This reason is used with the + // "BrokerRegistered" condition when it evaluates to False because the broker's + // pod re-registered under a different node_id than previously recorded — its + // data directory did not survive. This requires an operator decision (replace + // the broker) rather than silent adoption of the new identity. + BrokerBrokerRegisteredReasonIdentityChanged BrokerBrokerRegisteredCondition = "IdentityChanged" // BrokerBrokerRegisteredReasonError - This reason is used when a broker has // only been partially reconciled and we have early returned due to a retryable // error occurring prior to applying the desired broker state. @@ -914,6 +933,31 @@ const ( // has encountered a terminal error and will not be reconciled further. BrokerBrokerRegisteredReasonTerminalError BrokerBrokerRegisteredCondition = "TerminalError" + // BrokerConfigSynced - This condition indicates whether the broker's pod + // matches the desired pod template: both the config checksum and the + // restart-requiring cluster-config version. False means a rotation is pending + // (the pod will be recreated once the broker holds a valid roll-grant). + // + // This condition defaults to "Unknown" with a reason of "NotReconciled" and + // must be set by a controller when it subsequently reconciles a broker. + BrokerConfigSynced = "ConfigSynced" + // BrokerConfigSyncedReasonSynced - This reason is used with the "ConfigSynced" + // condition when it evaluates to True because the pod's config checksum and + // cluster-config version match the desired pod template. + BrokerConfigSyncedReasonSynced BrokerConfigSyncedCondition = "Synced" + // BrokerConfigSyncedReasonOutdated - This reason is used with the + // "ConfigSynced" condition when it evaluates to False because the pod's config + // checksum or cluster-config version differs from the desired pod template and + // the pod awaits rotation. + BrokerConfigSyncedReasonOutdated BrokerConfigSyncedCondition = "Outdated" + // BrokerConfigSyncedReasonError - This reason is used when a broker has only + // been partially reconciled and we have early returned due to a retryable error + // occurring prior to applying the desired broker state. + BrokerConfigSyncedReasonError BrokerConfigSyncedCondition = "Error" + // BrokerConfigSyncedReasonTerminalError - This reason is used when a broker has + // encountered a terminal error and will not be reconciled further. + BrokerConfigSyncedReasonTerminalError BrokerConfigSyncedCondition = "TerminalError" + // BrokerQuiesced - This condition indicates the broker is no longer reconciling // for its current generation. // @@ -931,21 +975,21 @@ const ( // BrokerStable - This condition is a roll-up status for automation (e.g. parent // controller checking all brokers are stable before proceeding). It is True - // when Ready, StorageBound, BrokerRegistered, and Quiesced all evaluate to - // True. Each tracks one dimension: Ready = pod health, StorageBound = PVC - // binding, BrokerRegistered = cluster membership, Quiesced = reconciliation - // complete. + // when Ready, StorageBound, BrokerRegistered, ConfigSynced, and Quiesced all + // evaluate to True. Each tracks one dimension: Ready = pod health, StorageBound + // = PVC binding, BrokerRegistered = cluster membership, ConfigSynced = no + // rotation pending, Quiesced = reconciliation complete. // // This condition defaults to "False" with a reason of "NotReconciled" and must // be set by a controller when it subsequently reconciles a broker. BrokerStable = "Stable" // BrokerStableReasonStable - This reason is used with the "Stable" condition - // when it evaluates to True because Ready, StorageBound, BrokerRegistered, and - // Quiesced all evaluate to True. + // when it evaluates to True because Ready, StorageBound, BrokerRegistered, + // ConfigSynced, and Quiesced all evaluate to True. BrokerStableReasonStable BrokerStableCondition = "Stable" // BrokerStableReasonUnstable - This reason is used with the "Stable" condition // when it evaluates to False because at least one of Ready, StorageBound, - // BrokerRegistered, or Quiesced evaluates to False. + // BrokerRegistered, ConfigSynced, or Quiesced evaluates to False. BrokerStableReasonUnstable BrokerStableCondition = "Unstable" ) @@ -2398,6 +2442,8 @@ type BrokerStatus struct { isStorageBoundTransientError bool isBrokerRegisteredSet bool isBrokerRegisteredTransientError bool + isConfigSyncedSet bool + isConfigSyncedTransientError bool } // NewBroker() returns a new BrokerStatus @@ -2553,6 +2599,8 @@ func (s *BrokerStatus) SetPodScheduled(reason BrokerPodScheduledCondition, messa status = metav1.ConditionTrue case BrokerPodScheduledReasonUnschedulable: status = metav1.ConditionFalse + case BrokerPodScheduledReasonPodMissing: + status = metav1.ConditionFalse case BrokerPodScheduledReasonError: s.isPodScheduledTransientError = true status = metav1.ConditionFalse @@ -2655,6 +2703,8 @@ func (s *BrokerStatus) SetBrokerRegistered(reason BrokerBrokerRegisteredConditio status = metav1.ConditionTrue case BrokerBrokerRegisteredReasonNotRegistered: status = metav1.ConditionFalse + case BrokerBrokerRegisteredReasonIdentityChanged: + status = metav1.ConditionFalse case BrokerBrokerRegisteredReasonError: s.isBrokerRegisteredTransientError = true status = metav1.ConditionFalse @@ -2677,9 +2727,60 @@ func (s *BrokerStatus) SetBrokerRegistered(reason BrokerBrokerRegisteredConditio }) } +// SetConfigSyncedFromCurrent sets the underlying condition based on an existing object. +func (s *BrokerStatus) SetConfigSyncedFromCurrent(o client.Object) { + condition := apimeta.FindStatusCondition(GetConditions(o), BrokerConfigSynced) + if condition == nil { + return + } + + s.SetConfigSynced(BrokerConfigSyncedCondition(condition.Reason), condition.Message) +} + +// SetConfigSynced sets the underlying condition to the given reason. +func (s *BrokerStatus) SetConfigSynced(reason BrokerConfigSyncedCondition, messages ...string) { + if s.isConfigSyncedSet { + panic("you should only ever set a condition once, doing so more than once is a programming error") + } + + var status metav1.ConditionStatus + + s.isConfigSyncedSet = true + message := strings.Join(messages, "; ") + + switch reason { + case BrokerConfigSyncedReasonSynced: + if message == "" { + message = "Pod config matches desired" + } + status = metav1.ConditionTrue + case BrokerConfigSyncedReasonOutdated: + status = metav1.ConditionFalse + case BrokerConfigSyncedReasonError: + s.isConfigSyncedTransientError = true + status = metav1.ConditionFalse + case BrokerConfigSyncedReasonTerminalError: + s.hasTerminalError = true + status = metav1.ConditionFalse + default: + panic("unhandled reason type") + } + + if message == "" { + panic("message must be set") + } + + s.conditions = append(s.conditions, metav1.Condition{ + Type: BrokerConfigSynced, + Status: status, + Reason: string(reason), + Message: message, + }) +} + func (s *BrokerStatus) getQuiesced() metav1.Condition { - transientErrorConditionsSet := s.isReadyTransientError || s.isPodScheduledTransientError || s.isStorageBoundTransientError || s.isBrokerRegisteredTransientError - allConditionsSet := s.isReadySet && s.isPodScheduledSet && s.isStorageBoundSet && s.isBrokerRegisteredSet + transientErrorConditionsSet := s.isReadyTransientError || s.isPodScheduledTransientError || s.isStorageBoundTransientError || s.isBrokerRegisteredTransientError || s.isConfigSyncedTransientError + allConditionsSet := s.isReadySet && s.isPodScheduledSet && s.isStorageBoundSet && s.isBrokerRegisteredSet && s.isConfigSyncedSet if (allConditionsSet || s.hasTerminalError) && !transientErrorConditionsSet { return metav1.Condition{ @@ -2700,7 +2801,7 @@ func (s *BrokerStatus) getQuiesced() metav1.Condition { func (s *BrokerStatus) getStable(conditions []metav1.Condition) metav1.Condition { allConditionsFoundAndTrue := true - for _, condition := range []string{BrokerReady, BrokerStorageBound, BrokerBrokerRegistered, BrokerQuiesced} { + for _, condition := range []string{BrokerReady, BrokerStorageBound, BrokerBrokerRegistered, BrokerConfigSynced, BrokerQuiesced} { conditionFoundAndTrue := false for _, setCondition := range conditions { if setCondition.Type == condition { diff --git a/operator/internal/statuses/zz_generated_status_test.go b/operator/internal/statuses/zz_generated_status_test.go index 825975290e..33e62683e4 100644 --- a/operator/internal/statuses/zz_generated_status_test.go +++ b/operator/internal/statuses/zz_generated_status_test.go @@ -1564,6 +1564,12 @@ func TestBroker(t *testing.T) { expected: metav1.ConditionFalse, setFn: func(status *BrokerStatus) { status.SetPodScheduled(BrokerPodScheduledReasonUnschedulable, "reason") }, }, + "PodScheduled/PodMissing": { + condition: BrokerPodScheduled, + reason: string(BrokerPodScheduledReasonPodMissing), + expected: metav1.ConditionFalse, + setFn: func(status *BrokerStatus) { status.SetPodScheduled(BrokerPodScheduledReasonPodMissing, "reason") }, + }, "PodScheduled/Error": { condition: BrokerPodScheduled, reason: string(BrokerPodScheduledReasonError), @@ -1616,6 +1622,14 @@ func TestBroker(t *testing.T) { status.SetBrokerRegistered(BrokerBrokerRegisteredReasonNotRegistered, "reason") }, }, + "BrokerRegistered/IdentityChanged": { + condition: BrokerBrokerRegistered, + reason: string(BrokerBrokerRegisteredReasonIdentityChanged), + expected: metav1.ConditionFalse, + setFn: func(status *BrokerStatus) { + status.SetBrokerRegistered(BrokerBrokerRegisteredReasonIdentityChanged, "reason") + }, + }, "BrokerRegistered/Error": { condition: BrokerBrokerRegistered, reason: string(BrokerBrokerRegisteredReasonError), @@ -1630,6 +1644,30 @@ func TestBroker(t *testing.T) { status.SetBrokerRegistered(BrokerBrokerRegisteredReasonTerminalError, "reason") }, }, + "ConfigSynced/Synced": { + condition: BrokerConfigSynced, + reason: string(BrokerConfigSyncedReasonSynced), + expected: metav1.ConditionTrue, + setFn: func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") }, + }, + "ConfigSynced/Outdated": { + condition: BrokerConfigSynced, + reason: string(BrokerConfigSyncedReasonOutdated), + expected: metav1.ConditionFalse, + setFn: func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonOutdated, "reason") }, + }, + "ConfigSynced/Error": { + condition: BrokerConfigSynced, + reason: string(BrokerConfigSyncedReasonError), + expected: metav1.ConditionFalse, + setFn: func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonError, "reason") }, + }, + "ConfigSynced/TerminalError": { + condition: BrokerConfigSynced, + reason: string(BrokerConfigSyncedReasonTerminalError), + expected: metav1.ConditionFalse, + setFn: func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonTerminalError, "reason") }, + }, } { tt := tt t.Run(name, func(t *testing.T) { @@ -1674,6 +1712,9 @@ func TestBroker(t *testing.T) { assertConditionStatusReason(t, conditionReason.condition, metav1.ConditionFalse, conditionReason.falseReason, status.getConditions(0)) status.SetBrokerRegistered(BrokerBrokerRegisteredReasonRegistered, "reason") + assertConditionStatusReason(t, conditionReason.condition, metav1.ConditionFalse, conditionReason.falseReason, status.getConditions(0)) + + status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") assertConditionStatusReason(t, conditionReason.condition, metav1.ConditionTrue, conditionReason.trueReason, status.getConditions(0)) }) } @@ -1691,6 +1732,7 @@ func TestBroker(t *testing.T) { func(status *BrokerStatus) { status.SetBrokerRegistered(BrokerBrokerRegisteredReasonRegistered, "reason") }, + func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") }, }, }, "Transient Error: Error, Condition: PodScheduled": { @@ -1701,6 +1743,7 @@ func TestBroker(t *testing.T) { func(status *BrokerStatus) { status.SetBrokerRegistered(BrokerBrokerRegisteredReasonRegistered, "reason") }, + func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") }, }, }, "Transient Error: Error, Condition: StorageBound": { @@ -1711,6 +1754,7 @@ func TestBroker(t *testing.T) { func(status *BrokerStatus) { status.SetBrokerRegistered(BrokerBrokerRegisteredReasonRegistered, "reason") }, + func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") }, }, }, "Transient Error: Error, Condition: BrokerRegistered": { @@ -1719,6 +1763,18 @@ func TestBroker(t *testing.T) { func(status *BrokerStatus) { status.SetReady(BrokerReadyReasonReady, "reason") }, func(status *BrokerStatus) { status.SetPodScheduled(BrokerPodScheduledReasonScheduled, "reason") }, func(status *BrokerStatus) { status.SetStorageBound(BrokerStorageBoundReasonBound, "reason") }, + func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") }, + }, + }, + "Transient Error: Error, Condition: ConfigSynced": { + setTransientErrFn: func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonError, "reason") }, + setConditionReasons: []setBrokerFunc{ + func(status *BrokerStatus) { status.SetReady(BrokerReadyReasonReady, "reason") }, + func(status *BrokerStatus) { status.SetPodScheduled(BrokerPodScheduledReasonScheduled, "reason") }, + func(status *BrokerStatus) { status.SetStorageBound(BrokerStorageBoundReasonBound, "reason") }, + func(status *BrokerStatus) { + status.SetBrokerRegistered(BrokerBrokerRegisteredReasonRegistered, "reason") + }, }, }, } { @@ -1747,6 +1803,7 @@ func TestBroker(t *testing.T) { "Terminal Error: TerminalError, Condition: BrokerRegistered": func(status *BrokerStatus) { status.SetBrokerRegistered(BrokerBrokerRegisteredReasonTerminalError, "reason") }, + "Terminal Error: TerminalError, Condition: ConfigSynced": func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonTerminalError, "reason") }, } { setFn := setFn t.Run(name, func(t *testing.T) { @@ -1781,6 +1838,7 @@ func TestBroker(t *testing.T) { func(status *BrokerStatus) { status.SetBrokerRegistered(BrokerBrokerRegisteredReasonRegistered, "reason") }, + func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") }, }, }, "Rollup Conditions: Stable, False Condition: Ready": { @@ -1794,6 +1852,7 @@ func TestBroker(t *testing.T) { func(status *BrokerStatus) { status.SetBrokerRegistered(BrokerBrokerRegisteredReasonRegistered, "reason") }, + func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") }, }, }, "Rollup Conditions: Stable, False Condition: StorageBound": { @@ -1807,6 +1866,7 @@ func TestBroker(t *testing.T) { func(status *BrokerStatus) { status.SetBrokerRegistered(BrokerBrokerRegisteredReasonRegistered, "reason") }, + func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") }, }, }, "Rollup Conditions: Stable, False Condition: BrokerRegistered": { @@ -1820,6 +1880,21 @@ func TestBroker(t *testing.T) { func(status *BrokerStatus) { status.SetReady(BrokerReadyReasonReady, "reason") }, func(status *BrokerStatus) { status.SetPodScheduled(BrokerPodScheduledReasonScheduled, "reason") }, func(status *BrokerStatus) { status.SetStorageBound(BrokerStorageBoundReasonBound, "reason") }, + func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonSynced, "reason") }, + }, + }, + "Rollup Conditions: Stable, False Condition: ConfigSynced": { + condition: BrokerStable, + trueReason: string(BrokerStableReasonStable), + falseReason: string(BrokerStableReasonUnstable), + falseCondition: func(status *BrokerStatus) { status.SetConfigSynced(BrokerConfigSyncedReasonTerminalError, "reason") }, + trueConditions: []setBrokerFunc{ + func(status *BrokerStatus) { status.SetReady(BrokerReadyReasonReady, "reason") }, + func(status *BrokerStatus) { status.SetPodScheduled(BrokerPodScheduledReasonScheduled, "reason") }, + func(status *BrokerStatus) { status.SetStorageBound(BrokerStorageBoundReasonBound, "reason") }, + func(status *BrokerStatus) { + status.SetBrokerRegistered(BrokerBrokerRegisteredReasonRegistered, "reason") + }, }, }, } { diff --git a/operator/pkg/feature/flags.go b/operator/pkg/feature/flags.go index 8e0691a3b4..a2b6a17e83 100644 --- a/operator/pkg/feature/flags.go +++ b/operator/pkg/feature/flags.go @@ -10,6 +10,11 @@ package feature import ( + "fmt" + "strconv" + "strings" + "time" + "github.com/redpanda-data/redpanda-operator/operator/cmd/syncclusterconfig" ) @@ -66,4 +71,63 @@ var ( Default: "additive", Parse: syncclusterconfig.StringToMode, }) + + // RollGrant is set by the cluster controller on a Broker CR to permit + // the Broker controller to perform a disruptive action on its pod + // (rotate, PV-affinity remediation). The cluster controller grants to at + // most one Broker at a time and revokes the grant once the roll + // completes. Value format: /. The + // checksum doubles as the grant's generation: a grant whose checksum does + // not match the Broker's desired pod template checksum is stale and + // rejected. + // Not registered in any bundle — has no default and is never auto-set. + RollGrant = &AnnotationFeatureFlag[string]{ + Key: "operator.redpanda.com/roll-grant", + Parse: func(s string) (string, error) { + return s, nil + }, + } + + // BrokerDeletionPolicy decides what happens to a Broker's pod and PVCs + // when its owning cluster is deleted: "cascade" (default) lets the GC + // delete them with the Broker CR — whole-cluster teardown — while + // "orphan" releases them (ownerRefs stripped, data survives). Users set + // it on the Cluster; the cluster controller propagates it onto each + // Broker CR so it stays readable after the Cluster object is gone. + // Deletion of a single Broker CR while its cluster is alive always + // releases the pod and PVCs, regardless of this policy. + // Not registered in any bundle — never auto-set. + BrokerDeletionPolicy = &AnnotationFeatureFlag[string]{ + Key: "operator.redpanda.com/broker-deletion-policy", + Default: "cascade", + Parse: func(s string) (string, error) { + return s, nil + }, + } ) + +// RollGrantTTL is the lease duration of a roll-grant. It is a safety valve +// against grants leaking across controller restarts or wedged rolls, not a +// pacing mechanism: an expired grant is treated as released and the cluster +// controller re-grants (health-gated) if the roll is still outstanding. +const RollGrantTTL = 10 * time.Minute + +// FormatRollGrant encodes a roll-grant annotation value, +// /. +func FormatRollGrant(checksum string, deadline time.Time) string { + return fmt.Sprintf("%s/%d", checksum, deadline.Unix()) +} + +// ParseRollGrant decodes a roll-grant annotation value. ok is false when the +// value is malformed; expiry is left to the caller. +func ParseRollGrant(grant string) (checksum string, deadline time.Time, ok bool) { + checksum, deadlineStr, found := strings.Cut(grant, "/") + if !found || checksum == "" { + return "", time.Time{}, false + } + unix, err := strconv.ParseInt(deadlineStr, 10, 64) + if err != nil { + return "", time.Time{}, false + } + return checksum, time.Unix(unix, 0), true +} diff --git a/operator/pkg/feature/flags_test.go b/operator/pkg/feature/flags_test.go new file mode 100644 index 0000000000..363e413b14 --- /dev/null +++ b/operator/pkg/feature/flags_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package feature + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRollGrantRoundTrip(t *testing.T) { + t.Parallel() + + deadline := time.Now().Add(RollGrantTTL).Truncate(time.Second) + grant := FormatRollGrant("abc123", deadline) + + checksum, parsed, ok := ParseRollGrant(grant) + require.True(t, ok) + assert.Equal(t, "abc123", checksum) + assert.True(t, parsed.Equal(deadline)) +} + +func TestParseRollGrantMalformed(t *testing.T) { + t.Parallel() + + cases := []string{ + "", // empty + "abc123", // no separator + "/1234", // empty checksum + "abc123/", // empty deadline + "abc123/later", // non-numeric deadline + } + for _, in := range cases { + _, _, ok := ParseRollGrant(in) + assert.Falsef(t, ok, "expected %q to be rejected", in) + } +} + +func TestParseRollGrantChecksumWithSlash(t *testing.T) { + t.Parallel() + + // The checksum must not contain "/" for the format to round-trip. Hex + // checksums never do; anything else fails the deadline parse and is + // rejected rather than silently truncated. + _, _, ok := ParseRollGrant("abc/def/1234") + assert.False(t, ok) +} diff --git a/operator/statuses.yaml b/operator/statuses.yaml index 6243594b9e..de2fc6847c 100644 --- a/operator/statuses.yaml +++ b/operator/statuses.yaml @@ -680,6 +680,10 @@ description: > This reason is used with the "PodScheduled" condition when it evaluates to False because the broker's pod cannot be scheduled (e.g. node affinity, insufficient resources). + - name: PodMissing + description: > + This reason is used with the "PodScheduled" condition when it evaluates to False because + the broker's pod does not exist (initial bootstrap, mid-rotation, or after decommission). - name: StorageBound description: > This condition indicates whether all PVCs for this broker are bound to PVs. @@ -716,6 +720,31 @@ description: > This reason is used with the "BrokerRegistered" condition when it evaluates to False because the broker has not yet registered with the cluster. + - name: IdentityChanged + description: > + This reason is used with the "BrokerRegistered" condition when it evaluates to False because + the broker's pod re-registered under a different node_id than previously recorded — its + data directory did not survive. This requires an operator decision (replace the broker) + rather than silent adoption of the new identity. + - name: ConfigSynced + description: > + This condition indicates whether the broker's pod matches the desired pod template: + both the config checksum and the restart-requiring cluster-config version. False means + a rotation is pending (the pod will be recreated once the broker holds a valid roll-grant). + + This condition defaults to "Unknown" with a reason of "NotReconciled" + and must be set by a controller when it subsequently reconciles a broker. + reasons: + - name: Synced + message: Pod config matches desired + description: > + This reason is used with the "ConfigSynced" condition when it evaluates to True because + the pod's config checksum and cluster-config version match the desired pod template. + - name: Outdated + description: > + This reason is used with the "ConfigSynced" condition when it evaluates to False because + the pod's config checksum or cluster-config version differs from the desired pod + template and the pod awaits rotation. - name: Quiesced final: true description: > @@ -735,13 +764,13 @@ This reason is used with the "Quiesced" condition when it evaluates to False because the operator has not finished reconciling the broker at its current generation. - name: Stable - rollup: [ Ready, StorageBound, BrokerRegistered, Quiesced ] + rollup: [ Ready, StorageBound, BrokerRegistered, ConfigSynced, Quiesced ] description: > This condition is a roll-up status for automation (e.g. parent controller checking all brokers are stable before proceeding). It is True when Ready, StorageBound, - BrokerRegistered, and Quiesced all evaluate to True. Each tracks one dimension: - Ready = pod health, StorageBound = PVC binding, BrokerRegistered = cluster - membership, Quiesced = reconciliation complete. + BrokerRegistered, ConfigSynced, and Quiesced all evaluate to True. Each tracks one + dimension: Ready = pod health, StorageBound = PVC binding, BrokerRegistered = cluster + membership, ConfigSynced = no rotation pending, Quiesced = reconciliation complete. This condition defaults to "False" with a reason of "NotReconciled" and must be set by a controller when it subsequently reconciles a broker. @@ -750,9 +779,10 @@ message: Broker stable description: > This reason is used with the "Stable" condition when it evaluates to True because - Ready, StorageBound, BrokerRegistered, and Quiesced all evaluate to True. + Ready, StorageBound, BrokerRegistered, ConfigSynced, and Quiesced all evaluate to True. - name: Unstable message: Broker unstable description: > This reason is used with the "Stable" condition when it evaluates to False because - at least one of Ready, StorageBound, BrokerRegistered, or Quiesced evaluates to False. \ No newline at end of file + at least one of Ready, StorageBound, BrokerRegistered, ConfigSynced, or Quiesced + evaluates to False. \ No newline at end of file From 98bf2813ef1c4de8cbf993fabdc8cfa405ff47e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Thu, 9 Jul 2026 12:06:02 +0200 Subject: [PATCH 02/29] operator: add the Broker CR reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the per-broker controller from the Broker CRD RFC: the "dumb executor" side of the separation of concerns. A sub-reconciler chain drives each Broker through PVC creation/adoption, unconditional pod-ensure (RFC Q5 — creation is never gated so bootstrap parallelizes and operator restarts can't strand a broker), PV-affinity remediation for dead nodes, roll-grant- gated pod rotation with leadership drain via maintenance mode, admin-API registration with node_id continuity verification (a rotated pod must rejoin under the same identity; a fresh identity is surfaced as Stuck + IdentityChanged rather than silently adopted), and decommission driven solely by the explicit spec.decommission intent, with recommission when the intent is unset mid-flight and the last-broker guard blocking both the spec and deletion paths (RFC Q2). Disruptive actions require a roll-grant issued by an owning cluster controller (not part of this change); this controller only consumes and never writes the annotation. Deleting a Broker CR without decommission intent releases its pod and PVCs while the owning cluster is alive; on cluster teardown the propagated broker-deletion-policy decides between cascade (default) and orphan, with any uncertainty resolving to orphan. Status reporting recomputes all conditions from the current pass — Ready, PodScheduled, StorageBound, BrokerRegistered, ConfigOutdated, Quiesced and the Stable roll-up — writes only on change, and classifies stuck pods (unschedulable, crash loops, image pull failures). Registered behind the --enable-broker flag (default false); no other controller is wired to it yet. Co-Authored-By: Claude Fable 5 --- .../files/rbac/v2-manager.ClusterRole.yaml | 23 + .../testdata/template-cases.golden.txtar | 2852 +++++++++++++++++ operator/cmd/run/run.go | 2 +- operator/config/rbac/bases/operator/role.yaml | 12 +- operator/config/rbac/itemized/v2-manager.yaml | 23 + .../controller/pvcunbinder/pvcunbinder.go | 90 +- .../pvcunbinder/pvcunbinder_internal_test.go | 6 +- .../controller/redpanda/broker_controller.go | 1069 +++++- .../redpanda/nodepool_controller.go | 14 +- .../redpanda/redpanda_controller.go | 5 +- .../controller/redpanda/testdata/role.yaml | 20 +- 11 files changed, 4055 insertions(+), 61 deletions(-) diff --git a/operator/chart/files/rbac/v2-manager.ClusterRole.yaml b/operator/chart/files/rbac/v2-manager.ClusterRole.yaml index fe7ddbac2a..de1fb5a793 100644 --- a/operator/chart/files/rbac/v2-manager.ClusterRole.yaml +++ b/operator/chart/files/rbac/v2-manager.ClusterRole.yaml @@ -9,6 +9,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -36,6 +37,20 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -261,3 +276,11 @@ rules: - patch - update - watch + - apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch diff --git a/operator/chart/testdata/template-cases.golden.txtar b/operator/chart/testdata/template-cases.golden.txtar index ec17a00a68..6e1a50ce7b 100644 --- a/operator/chart/testdata/template-cases.golden.txtar +++ b/operator/chart/testdata/template-cases.golden.txtar @@ -327,6 +327,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -354,6 +355,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -579,6 +594,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1219,6 +1242,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -1246,6 +1270,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -1471,6 +1509,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1890,6 +1936,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -1917,6 +1964,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -2142,6 +2203,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -2809,6 +2878,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -2836,6 +2906,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -3061,6 +3145,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -3500,6 +3592,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -3527,6 +3620,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -3752,6 +3859,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -4405,6 +4520,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -4432,6 +4548,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -4657,6 +4787,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -5240,6 +5378,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -5267,6 +5406,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -5492,6 +5645,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -6348,6 +6509,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -6375,6 +6537,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -6600,6 +6776,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -7124,6 +7308,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -7151,6 +7336,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -7376,6 +7575,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -8030,6 +8237,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -8057,6 +8265,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -8282,6 +8504,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -8705,6 +8935,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -8732,6 +8963,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -8957,6 +9202,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -9649,6 +9902,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -9676,6 +9930,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -9901,6 +10169,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -10412,6 +10688,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -10439,6 +10716,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -10664,6 +10955,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -11402,6 +11701,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -11429,6 +11729,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -11654,6 +11968,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -12198,6 +12520,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -12225,6 +12548,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -12450,6 +12787,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -13121,6 +13466,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -13148,6 +13494,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -13373,6 +13733,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -13799,6 +14167,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -13826,6 +14195,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -14051,6 +14434,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -14710,6 +15101,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -14737,6 +15129,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -14962,6 +15368,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -15390,6 +15804,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -15417,6 +15832,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -15642,6 +16071,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -16333,6 +16770,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -16360,6 +16798,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -16585,6 +17037,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -17011,6 +17471,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -17038,6 +17499,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -17263,6 +17738,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -17762,6 +18245,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -17789,6 +18273,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -18014,6 +18512,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -18454,6 +18960,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -18481,6 +18988,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -18706,6 +19227,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -19372,6 +19901,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -19399,6 +19929,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -19624,6 +20168,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -20226,6 +20778,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -20253,6 +20806,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -20478,6 +21045,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -21351,6 +21926,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -21378,6 +21954,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -21603,6 +22193,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -22038,6 +22636,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -22065,6 +22664,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -22290,6 +22903,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -22835,6 +23456,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -22862,6 +23484,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -23087,6 +23723,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -23501,6 +24145,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -23528,6 +24173,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -23753,6 +24412,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -24421,6 +25088,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -24448,6 +25116,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -24673,6 +25355,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -25107,6 +25797,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -25134,6 +25825,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -25359,6 +26064,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -26019,6 +26732,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -26046,6 +26760,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -26271,6 +26999,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -27072,6 +27808,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -27099,6 +27836,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -27324,6 +28075,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -28201,6 +28960,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -28228,6 +28988,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -28453,6 +29227,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -28888,6 +29670,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -28915,6 +29698,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -29140,6 +29937,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -29839,6 +30644,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -29866,6 +30672,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -30091,6 +30911,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -30675,6 +31503,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -30702,6 +31531,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -30927,6 +31770,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -31743,6 +32594,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -31770,6 +32622,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -31995,6 +32861,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -32800,6 +33674,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -32827,6 +33702,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -33052,6 +33941,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -33867,6 +34764,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -33894,6 +34792,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -34119,6 +35031,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -34705,6 +35625,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -34732,6 +35653,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -34957,6 +35892,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -36012,6 +36955,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -36039,6 +36983,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -36264,6 +37222,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -36698,6 +37664,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -36725,6 +37692,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -36950,6 +37931,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -37622,6 +38611,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -37649,6 +38639,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -37874,6 +38878,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -38475,6 +39487,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -38502,6 +39515,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -38727,6 +39754,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -39708,6 +40743,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -39735,6 +40771,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -39960,6 +41010,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -40747,6 +41805,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -40774,6 +41833,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -40999,6 +42072,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -41862,6 +42943,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -41889,6 +42971,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -42114,6 +43210,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -42968,6 +44072,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -42995,6 +44100,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -43220,6 +44339,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -44107,6 +45234,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -44134,6 +45262,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -44359,6 +45501,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -44804,6 +45954,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -44831,6 +45982,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -45056,6 +46221,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -45929,6 +47102,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -45956,6 +47130,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -46181,6 +47369,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -46775,6 +47971,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -46802,6 +47999,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -47027,6 +48238,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -47929,6 +49148,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -47956,6 +49176,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -48181,6 +49415,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -49305,6 +50547,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -49332,6 +50575,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -49557,6 +50814,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -50614,6 +51879,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -50641,6 +51907,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -50866,6 +52146,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -51411,6 +52699,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -51438,6 +52727,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -51663,6 +52966,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -52357,6 +53668,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -52384,6 +53696,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -52609,6 +53935,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -53197,6 +54531,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -53224,6 +54559,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -53449,6 +54798,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -54575,6 +55932,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -54602,6 +55960,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -54827,6 +56199,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -56992,6 +58372,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -57019,6 +58400,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -57244,6 +58639,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -58474,6 +59877,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -58501,6 +59905,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -58726,6 +60144,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -60124,6 +61550,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -60151,6 +61578,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -60376,6 +61817,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -61342,6 +62791,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -61369,6 +62819,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -61594,6 +63058,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -63726,6 +65198,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -63753,6 +65226,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -63978,6 +65465,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -65182,6 +66677,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -65209,6 +66705,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -65434,6 +66944,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -65978,6 +67496,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -66005,6 +67524,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -66230,6 +67763,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -67471,6 +69012,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -67498,6 +69040,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -67723,6 +69279,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -68939,6 +70503,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -68966,6 +70531,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -69191,6 +70770,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -70151,6 +71738,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -70178,6 +71766,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -70403,6 +72005,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -71703,6 +73313,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -71730,6 +73341,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -71955,6 +73580,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -73152,6 +74785,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -73179,6 +74813,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -73404,6 +75052,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -73969,6 +75625,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -73996,6 +75653,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -74221,6 +75892,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -75232,6 +76911,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -75259,6 +76939,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -75484,6 +77178,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -76217,6 +77919,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -76244,6 +77947,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -76469,6 +78186,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -77720,6 +79445,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -77747,6 +79473,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -77972,6 +79712,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -78516,6 +80264,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -78543,6 +80292,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -78768,6 +80531,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -79922,6 +81693,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -79949,6 +81721,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -80174,6 +81960,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -80721,6 +82515,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -80748,6 +82543,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -80973,6 +82782,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -81614,6 +83431,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -81641,6 +83459,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -81866,6 +83698,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -82285,6 +84125,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -82312,6 +84153,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -82537,6 +84392,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -83220,6 +85083,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -83247,6 +85111,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -83472,6 +85350,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -83982,6 +85868,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -84009,6 +85896,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -84234,6 +86135,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -84917,6 +86826,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -84944,6 +86854,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -85169,6 +87093,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -85678,6 +87610,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -85705,6 +87638,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -85930,6 +87877,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -86569,6 +88524,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -86596,6 +88552,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -86821,6 +88791,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -87240,6 +89218,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -87267,6 +89246,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -87492,6 +89485,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -88148,6 +90149,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -88175,6 +90177,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -88400,6 +90416,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -88819,6 +90843,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -88846,6 +90871,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -89071,6 +91110,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -89710,6 +91757,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -89737,6 +91785,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -89962,6 +92024,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -90381,6 +92451,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -90408,6 +92479,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -90633,6 +92718,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -91272,6 +93365,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -91299,6 +93393,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -91524,6 +93632,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -91943,6 +94059,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -91970,6 +94087,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -92195,6 +94326,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -92834,6 +94973,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -92861,6 +95001,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -93086,6 +95240,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -93505,6 +95667,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -93532,6 +95695,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -93757,6 +95934,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -94404,6 +96589,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -94431,6 +96617,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -94656,6 +96856,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -95075,6 +97283,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -95102,6 +97311,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -95327,6 +97550,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -95974,6 +98205,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -96001,6 +98233,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -96226,6 +98472,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -96645,6 +98899,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -96672,6 +98927,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -96897,6 +99166,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -97570,6 +99847,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -97597,6 +99875,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -97822,6 +100114,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -98241,6 +100541,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -98268,6 +100569,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -98493,6 +100808,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -99256,6 +101579,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -99283,6 +101607,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -99508,6 +101846,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -99927,6 +102273,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -99954,6 +102301,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -100179,6 +102540,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -100854,6 +103223,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -100881,6 +103251,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -101106,6 +103490,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -101525,6 +103917,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -101552,6 +103945,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -101777,6 +104184,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -102452,6 +104867,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -102479,6 +104895,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -102704,6 +105134,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -103123,6 +105561,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -103150,6 +105589,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -103375,6 +105828,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -104049,6 +106510,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -104076,6 +106538,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -104301,6 +106777,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -104736,6 +107220,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -104763,6 +107248,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -104988,6 +107487,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -105732,6 +108239,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -105759,6 +108267,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -105984,6 +108506,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -106510,6 +109040,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -106537,6 +109068,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -106762,6 +109307,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -107561,6 +110114,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -107588,6 +110142,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -107813,6 +110381,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -108339,6 +110915,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -108366,6 +110943,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -108591,6 +111182,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -109384,6 +111983,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -109411,6 +112011,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -109636,6 +112250,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -110162,6 +112784,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -110189,6 +112812,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -110414,6 +113051,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -111140,6 +113785,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -111167,6 +113813,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -111392,6 +114052,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -111918,6 +114586,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -111945,6 +114614,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -112170,6 +114853,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -112888,6 +115579,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -112915,6 +115607,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -113140,6 +115846,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -113650,6 +116364,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -113677,6 +116392,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -113902,6 +116631,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -114672,6 +117409,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -114699,6 +117437,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -114924,6 +117676,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -115343,6 +118103,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -115370,6 +118131,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -115595,6 +118370,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -116241,6 +119024,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -116268,6 +119052,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -116493,6 +119291,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -116912,6 +119718,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -116939,6 +119746,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -117164,6 +119985,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -117804,6 +120633,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -117831,6 +120661,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -118056,6 +120900,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/operator/cmd/run/run.go b/operator/cmd/run/run.go index 323271a9cf..36063136f3 100644 --- a/operator/cmd/run/run.go +++ b/operator/cmd/run/run.go @@ -571,7 +571,7 @@ func Run( } if opts.enableBrokerController { - if err := redpandacontrollers.SetupBrokerController(ctx, mcmanager, opts.namespace); err != nil { + if err := redpandacontrollers.SetupBrokerController(ctx, mcmanager, factory, opts.namespace, opts.unbindPVCsAfter); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Broker") return err } diff --git a/operator/config/rbac/bases/operator/role.yaml b/operator/config/rbac/bases/operator/role.yaml index f80489aece..16fcc388ba 100644 --- a/operator/config/rbac/bases/operator/role.yaml +++ b/operator/config/rbac/bases/operator/role.yaml @@ -9,6 +9,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -30,17 +31,6 @@ rules: - get - list - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - delete - - get - - list - - patch - - update - - watch - apiGroups: - "" resources: diff --git a/operator/config/rbac/itemized/v2-manager.yaml b/operator/config/rbac/itemized/v2-manager.yaml index 4b017820e0..2c444041a8 100644 --- a/operator/config/rbac/itemized/v2-manager.yaml +++ b/operator/config/rbac/itemized/v2-manager.yaml @@ -9,6 +9,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -36,6 +37,20 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch - apiGroups: - apps resources: @@ -261,3 +276,11 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch diff --git a/operator/internal/controller/pvcunbinder/pvcunbinder.go b/operator/internal/controller/pvcunbinder/pvcunbinder.go index 5038a07885..029de568aa 100644 --- a/operator/internal/controller/pvcunbinder/pvcunbinder.go +++ b/operator/internal/controller/pvcunbinder/pvcunbinder.go @@ -41,7 +41,11 @@ import ( "github.com/redpanda-data/redpanda-operator/pkg/multicluster" ) -var schedulingFailureRE = regexp.MustCompile(`(^0/[1-9]\d* nodes are available)|(volume node affinity)`) +// SchedulingFailureRE matches scheduler messages that indicate a Pod +// cannot be placed because of volume node-affinity constraints. Used by +// both the PVCUnbinder and the Broker controller to detect dead-node +// scenarios. +var SchedulingFailureRE = regexp.MustCompile(`(^0/[1-9]\d* nodes are available)|(volume node affinity)`) // PauseAnnotation - when present and set to "true" on the parent // Redpanda or Cluster CR, instructs the PVCUnbinder to skip all @@ -871,7 +875,7 @@ func (r *Controller) freedPVBlocking(ctx context.Context, pv *corev1.PersistentV return false, nil } - hostname := nodeFromPVAffinity(pv) + hostname := NodeFromPVAffinity(pv) if hostname == "" { // Can't resolve the pinned node; be conservative — an // Available freed PV we can't classify is treated as a live @@ -920,7 +924,7 @@ func (r *Controller) ShouldRemediate(ctx context.Context, pod *corev1.Pod) (bool // As of Kubernetes >1.21.x <=1.28.x (Didn't track down an exact version), // volume node affinity conflicts no longer seem to appear in the message, // hence the need to check for a much weaker case. - if !schedulingFailureRE.MatchString(cond.Message) { + if !SchedulingFailureRE.MatchString(cond.Message) { log.FromContext(ctx).Info("scheduling failure does not appear to indicate volume affinity issues; skipping", "name", pod.Name, "condition", cond) return false, 0 } @@ -1064,7 +1068,7 @@ func (r *Controller) multiNodeEventInProgress(ctx context.Context) (bool, error) if pv.Spec.ClaimRef == nil { continue } - hostname := nodeFromPVAffinity(pv) + hostname := NodeFromPVAffinity(pv) if hostname == "" { continue } @@ -1118,7 +1122,7 @@ func (r *Controller) multiNodeEventInProgress(ctx context.Context) (bool, error) if !pvcUnbinderPredicate(other) { continue } - if !podHasVolumeAffinityUnschedulable(other) { + if !PodHasVolumeAffinityUnschedulable(other) { continue } for _, pvcKey := range StsPVCs(other) { @@ -1133,7 +1137,7 @@ func (r *Controller) multiNodeEventInProgress(ctx context.Context) (bool, error) return false, nil } -// nodeFromPVAffinity extracts the hostname value pinned by a PV's +// NodeFromPVAffinity extracts the hostname value pinned by a PV's // NodeAffinity, used by Gate 2 to bucket stuck pods by their pinned // node. Only `kubernetes.io/hostname` `In` selectors with a single // value are recognized — that's the shape Local / HostPath volumes @@ -1147,7 +1151,9 @@ func (r *Controller) multiNodeEventInProgress(ctx context.Context) (bool, error) // Gate 2 is best-effort regardless: any PV we can't classify just // doesn't contribute to the count. The binding safety invariant rests // on Gates 0/3/4, not on this function's coverage. -func nodeFromPVAffinity(pv *corev1.PersistentVolume) string { +// NodeFromPVAffinity extracts the hostname from a PV's NodeAffinity. +// Returns "" if no single-hostname affinity is found. +func NodeFromPVAffinity(pv *corev1.PersistentVolume) string { if pv.Spec.NodeAffinity == nil || pv.Spec.NodeAffinity.Required == nil { return "" } @@ -1167,6 +1173,64 @@ func nodeFromPVAffinity(pv *corev1.PersistentVolume) string { return "" } +// DeadNodePVCs returns the PVCs attached to the pod whose bound PVs are +// pinned (HostPath or Local volume with NodeAffinity) to nodes that no +// longer exist. As a side effect, each affected PV's reclaim policy is +// patched to Retain so the storage survives PVC deletion. +// +// apiReader must be an uncached client for accurate Node existence checks. +func DeadNodePVCs(ctx context.Context, c client.Client, apiReader client.Reader, pod *corev1.Pod) ([]corev1.PersistentVolumeClaim, error) { + l := log.FromContext(ctx) + var affected []corev1.PersistentVolumeClaim + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].PersistentVolumeClaim == nil { + continue + } + var pvc corev1.PersistentVolumeClaim + if err := c.Get(ctx, client.ObjectKey{ + Name: pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName, + Namespace: pod.Namespace, + }, &pvc); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, err + } + if pvc.Spec.VolumeName == "" { + continue + } + var pv corev1.PersistentVolume + if err := c.Get(ctx, client.ObjectKey{Name: pvc.Spec.VolumeName}, &pv); err != nil { + return nil, err + } + if pv.Spec.HostPath == nil && pv.Spec.Local == nil { + continue + } + nodeName := NodeFromPVAffinity(&pv) + if nodeName == "" { + continue + } + var node corev1.Node + if err := apiReader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { + if !apierrors.IsNotFound(err) { + return nil, err + } + } else { + continue + } + if pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + patch := client.StrategicMergeFrom(pv.DeepCopy(), &client.MergeFromWithOptimisticLock{}) + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain + if err := c.Patch(ctx, &pv, patch); err != nil { + return nil, fmt.Errorf("patching PV %s to Retain: %w", pv.Name, err) + } + l.Info("patched PV to Retain", "pv", pv.Name, "deadNode", nodeName) + } + affected = append(affected, pvc) + } + return affected, nil +} + // listClusterPVCsByName returns a name→PVC snapshot for the PVCs that // belong to the same Redpanda/Cluster as `pod` (matched by the // app.kubernetes.io/instance label). Gate 3 inspects spec.volumeName @@ -1194,16 +1258,16 @@ func (r *Controller) listClusterPVCsByName(ctx context.Context, pod *corev1.Pod) return out, nil } -// podHasVolumeAffinityUnschedulable reports whether a Pod is Pending -// for the same reason that would cause the unbinder to act on it: the -// scheduler couldn't satisfy volume node affinity. Used by -// [Controller.multiNodeEventInProgress] to detect K8s-wide events. -func podHasVolumeAffinityUnschedulable(pod *corev1.Pod) bool { +// PodHasVolumeAffinityUnschedulable reports whether a Pod is Pending +// because the scheduler couldn't satisfy volume node affinity. Used by +// [Controller.multiNodeEventInProgress] and by the Broker controller to +// detect dead-node scenarios. +func PodHasVolumeAffinityUnschedulable(pod *corev1.Pod) bool { for _, cond := range pod.Status.Conditions { if cond.Type != corev1.PodScheduled || cond.Status != corev1.ConditionFalse || cond.Reason != "Unschedulable" { continue } - return schedulingFailureRE.MatchString(cond.Message) + return SchedulingFailureRE.MatchString(cond.Message) } return false } diff --git a/operator/internal/controller/pvcunbinder/pvcunbinder_internal_test.go b/operator/internal/controller/pvcunbinder/pvcunbinder_internal_test.go index 9b14b62c8d..54d9bf031a 100644 --- a/operator/internal/controller/pvcunbinder/pvcunbinder_internal_test.go +++ b/operator/internal/controller/pvcunbinder/pvcunbinder_internal_test.go @@ -572,12 +572,12 @@ func TestPodHasVolumeAffinityUnschedulable(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { pod := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{tc.condition}}} - require.Equal(t, tc.want, podHasVolumeAffinityUnschedulable(pod)) + require.Equal(t, tc.want, PodHasVolumeAffinityUnschedulable(pod)) }) } t.Run("no PodScheduled condition", func(t *testing.T) { - require.False(t, podHasVolumeAffinityUnschedulable(&corev1.Pod{})) + require.False(t, PodHasVolumeAffinityUnschedulable(&corev1.Pod{})) }) } @@ -969,7 +969,7 @@ func TestNodeFromPVAffinity(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.want, nodeFromPVAffinity(tc.pv)) + require.Equal(t, tc.want, NodeFromPVAffinity(tc.pv)) }) } } diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index 893ddb0528..97d5874d01 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -11,39 +11,123 @@ package redpanda import ( "context" + "fmt" + "strings" + "time" + "github.com/go-logr/logr" "github.com/redpanda-data/common-go/otelutil/log" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + ctrlcontroller "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" redpandav1alpha2 "github.com/redpanda-data/redpanda-operator/operator/api/redpanda/v1alpha2" + vectorizedv1alpha1 "github.com/redpanda-data/redpanda-operator/operator/api/vectorized/v1alpha1" "github.com/redpanda-data/redpanda-operator/operator/internal/controller" + "github.com/redpanda-data/redpanda-operator/operator/internal/controller/pvcunbinder" "github.com/redpanda-data/redpanda-operator/operator/internal/observability" + "github.com/redpanda-data/redpanda-operator/operator/internal/statuses" + internalclient "github.com/redpanda-data/redpanda-operator/operator/pkg/client" + "github.com/redpanda-data/redpanda-operator/operator/pkg/feature" "github.com/redpanda-data/redpanda-operator/pkg/multicluster" ) // +kubebuilder:rbac:groups=cluster.redpanda.com,resources=brokers,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cluster.redpanda.com,resources=brokers/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cluster.redpanda.com,resources=brokers/finalizers,verbs=update +// +kubebuilder:rbac:groups=redpanda.vectorized.io,resources=clusters,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get;list;patch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get + +const brokerFinalizerName = "cluster.redpanda.com/broker-decommission" + +const requeueShort = 2 * time.Second + +// requeueDrain paces the leadership-drain poll during a granted rotation. +// Deliberately much shorter than periodicRequeue: rolls are serialized +// cluster-wide by the roll-grant, so drain-wait latency accumulates across +// every broker in the fleet. +const requeueDrain = 10 * time.Second type BrokerReconciler struct { - Manager multicluster.Manager + Manager multicluster.Manager + ClientFactory internalclient.ClientFactory + // UnbindPVCsAfter is the duration a pod must be stuck in Pending + // with volume node-affinity conflict before PVC remediation fires. + // Zero disables the remediation. + UnbindPVCsAfter time.Duration +} + +func SetupBrokerController(_ context.Context, mgr multicluster.Manager, clientFactory internalclient.ClientFactory, namespace string, unbindPVCsAfter time.Duration) error { + return mcbuilder.ControllerManagedBy(mgr).WithOptions(ctrlcontroller.TypedOptions[mcreconcile.Request]{ + SkipNameValidation: ptr.To(true), + }).For( + &redpandav1alpha2.Broker{}, + mcbuilder.WithEngageWithLocalCluster(true), + mcbuilder.WithEngageWithProviderClusters(true), + ). + Owns(&corev1.Pod{}, mcbuilder.WithEngageWithLocalCluster(true), mcbuilder.WithEngageWithProviderClusters(true)). + Owns(&corev1.PersistentVolumeClaim{}, mcbuilder.WithEngageWithLocalCluster(true), mcbuilder.WithEngageWithProviderClusters(true)). + Complete( + controller.FilterNamespaceReconciler( + namespace, + observability.Wrap[mcreconcile.Request](&BrokerReconciler{ + Manager: mgr, + ClientFactory: clientFactory, + UnbindPVCsAfter: unbindPVCsAfter, + }, "Broker", periodicRequeue))) } -func SetupBrokerController(_ context.Context, mgr multicluster.Manager, namespace string) error { - return mcbuilder.ControllerManagedBy(mgr). - For( - &redpandav1alpha2.Broker{}, - mcbuilder.WithEngageWithLocalCluster(true), - mcbuilder.WithEngageWithProviderClusters(true), - ).Complete( - controller.FilterNamespaceReconciler( - namespace, - observability.Wrap[mcreconcile.Request](&BrokerReconciler{ - Manager: mgr, - }, "Broker", periodicRequeue))) +type brokerReconciliationState struct { + broker *redpandav1alpha2.Broker + pod *corev1.Pod // nil when pod does not exist yet + phase redpandav1alpha2.BrokerPhase // empty = compute from pod status + granted bool + // initialStatus snapshots Status as fetched, so syncBrokerStatus can + // skip the API write when nothing changed (RFC Q11: rate-limit status + // updates). + initialStatus *redpandav1alpha2.BrokerStatus + // registrationVerified is set when THIS reconcile confirmed via the + // admin API that the pod is registered under the expected node_id. The + // BrokerRegistered condition mirrors it, so it never reports stale + // pre-rotation state. + registrationVerified bool + // registrationConflict carries the identity-mismatch message when the + // pod re-registered under an unexpected node_id. + registrationConflict string +} + +type brokerReconcilerFn func(ctx context.Context, state *brokerReconciliationState, cluster cluster.Cluster) (ctrl.Result, error) + +func (r *BrokerReconciler) fetchState(ctx context.Context, k8sClient client.Client, broker *redpandav1alpha2.Broker) (*brokerReconciliationState, error) { + state := &brokerReconciliationState{ + broker: broker, + granted: hasValidRollGrant(ctx, broker), + initialStatus: broker.Status.DeepCopy(), + } + + var pod corev1.Pod + err := k8sClient.Get(ctx, client.ObjectKey{Name: broker.PodName(), Namespace: broker.Namespace}, &pod) + switch { + case apierrors.IsNotFound(err): + // state.pod remains nil + case err != nil: + return nil, err + default: + state.pod = &pod + } + + return state, nil } func (r *BrokerReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { @@ -55,10 +139,965 @@ func (r *BrokerReconciler) Reconcile(ctx context.Context, req mcreconcile.Reques return ctrl.Result{}, err } - broker := &redpandav1alpha2.Broker{} - if err := k8sCluster.GetClient().Get(ctx, req.NamespacedName, broker); err != nil { + k8sClient := k8sCluster.GetClient() + + var broker redpandav1alpha2.Broker + if err := k8sClient.Get(ctx, req.NamespacedName, &broker); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + if broker.DeletionTimestamp.IsZero() { + if !controllerutil.ContainsFinalizer(&broker, brokerFinalizerName) { + controllerutil.AddFinalizer(&broker, brokerFinalizerName) + if err := k8sClient.Update(ctx, &broker); err != nil { + return ctrl.Result{}, err + } + } + } else { + return r.reconcileDelete(ctx, l, k8sClient, &broker, broker.PodName()) + } + + state, err := r.fetchState(ctx, k8sClient, &broker) + if err != nil { + return ctrl.Result{}, err + } + + reconcilers := []brokerReconcilerFn{ + r.reconcilePVCs, + r.reconcilePod, + r.reconcilePVCAdoption, + r.reconcilePVAffinity, + r.reconcilePodRotation, + r.reconcileBrokerRegistration, + r.reconcileDecommission, + } + + for _, fn := range reconcilers { + result, err := fn(ctx, state, k8sCluster) + if err != nil { + return ctrl.Result{}, err + } + if !result.IsZero() { + return r.syncBrokerStatus(ctx, state, k8sCluster, result) + } + } + + return r.syncBrokerStatus(ctx, state, k8sCluster, ctrl.Result{}) +} + +func (r *BrokerReconciler) reconcilePVCs(ctx context.Context, state *brokerReconciliationState, cluster cluster.Cluster) (ctrl.Result, error) { + broker := state.broker + if broker.Spec.Decommission { + return ctrl.Result{}, nil + } + + l := log.FromContext(ctx) + k8sClient := cluster.GetClient() + scheme := cluster.GetScheme() + podName := broker.PodName() + + for _, vct := range broker.Spec.Storage.VolumeClaimTemplates { + pvcName := fmt.Sprintf("%s-%s", vct.Name, podName) + var pvc corev1.PersistentVolumeClaim + if err := k8sClient.Get(ctx, client.ObjectKey{Name: pvcName, Namespace: broker.Namespace}, &pvc); err == nil { + // A PVC mid-deletion (e.g. PV-affinity remediation) must be fully + // gone before recreating it or the pod: pvc-protection releases a + // Terminating PVC only once no pod references it, so recreating + // the pod first pins the old PVC forever and the pod never + // schedules. + if !pvc.DeletionTimestamp.IsZero() { + l.Info("waiting for PVC deletion to complete", "pvc", pvcName) + return ctrl.Result{RequeueAfter: requeueShort}, nil + } + } else { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + pvc = corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: broker.Namespace, + Labels: broker.Spec.PodTemplate.Labels, + }, + Spec: vct.Spec, + } + if err := controllerutil.SetControllerReference(broker, &pvc, scheme); err != nil { + return ctrl.Result{}, err + } + l.Info("creating PVC", "name", pvcName) + if err := k8sClient.Create(ctx, &pvc); err != nil { + return ctrl.Result{}, err + } + } + } + return ctrl.Result{}, nil +} + +func (r *BrokerReconciler) reconcilePod(ctx context.Context, state *brokerReconciliationState, cluster cluster.Cluster) (ctrl.Result, error) { + l := log.FromContext(ctx) + k8sClient := cluster.GetClient() + scheme := cluster.GetScheme() + broker := state.broker + podName := broker.PodName() + + if state.pod == nil { + if broker.Spec.Decommission { + return ctrl.Result{}, nil + } + // Pod-ensure is deliberately NOT gated on a roll-grant (RFC Q5): + // creation is non-disruptive, initial cluster bootstrap needs all + // pods in parallel, and an expired grant must never strand a broker + // between rotation's delete and the recreate. Only disruptive + // actions (rotation, PV remediation) require a grant. + l.Info("creating pod (no existing pod found)", "name", podName) + newPod := broker.BuildPod(podName) + if err := controllerutil.SetControllerReference(broker, newPod, scheme); err != nil { + return ctrl.Result{}, err + } + if err := k8sClient.Create(ctx, newPod); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + pod := state.pod + ownerRef := metav1.GetControllerOf(pod) + if ownerRef != nil && !metav1.IsControlledBy(pod, broker) { + l.Info("pod owned by another controller (shadow mode)", "owner", ownerRef.Kind+"/"+ownerRef.Name) + state.phase = redpandav1alpha2.BrokerPhasePending + return ctrl.Result{RequeueAfter: periodicRequeue}, nil + } + if ownerRef == nil { + l.Info("adopting orphaned pod", "name", podName) + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + if cs := broker.Spec.PodTemplate.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation]; cs != "" { + pod.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation] = cs + } + if err := controllerutil.SetControllerReference(broker, pod, scheme); err != nil { + return ctrl.Result{}, err + } + if err := k8sClient.Update(ctx, pod); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + return ctrl.Result{}, nil +} + +func (r *BrokerReconciler) reconcilePVCAdoption(ctx context.Context, state *brokerReconciliationState, cluster cluster.Cluster) (ctrl.Result, error) { + if state.pod == nil { + return ctrl.Result{}, nil + } + l := log.FromContext(ctx) + k8sClient := cluster.GetClient() + scheme := cluster.GetScheme() + broker := state.broker + + for _, ec := range broker.Spec.Storage.ExistingClaims { + var pvc corev1.PersistentVolumeClaim + if err := k8sClient.Get(ctx, client.ObjectKey{Name: ec.Name, Namespace: broker.Namespace}, &pvc); err != nil { + l.Info("could not get PVC for adoption", "name", ec.Name, "error", err) + continue + } + if metav1.GetControllerOf(&pvc) != nil { + continue + } + if err := controllerutil.SetControllerReference(broker, &pvc, scheme); err != nil { + return ctrl.Result{}, err + } + if err := k8sClient.Update(ctx, &pvc); err != nil { + return ctrl.Result{}, err + } + l.Info("adopted PVC", "name", ec.Name) + } + return ctrl.Result{}, nil +} + +func (r *BrokerReconciler) reconcilePVAffinity(ctx context.Context, state *brokerReconciliationState, cluster cluster.Cluster) (ctrl.Result, error) { + if state.pod == nil || !pvcunbinder.PodHasVolumeAffinityUnschedulable(state.pod) { + return ctrl.Result{}, nil + } + l := log.FromContext(ctx) + if !state.granted { + l.Info("pod stuck on PV node affinity, waiting for roll-grant", "name", state.pod.Name) + state.phase = redpandav1alpha2.BrokerPhaseStuck + return ctrl.Result{RequeueAfter: periodicRequeue}, nil + } + k8sClient := cluster.GetClient() + apiReader := cluster.GetAPIReader() + remediated, err := r.remediatePVAffinity(ctx, l, k8sClient, apiReader, state.broker, state.pod) + if err != nil { + return ctrl.Result{}, err + } + if remediated { + // Remediation discarded the broker's storage: the replacement pod + // starts with an empty data dir and registers under a fresh + // node_id. Clear the recorded identity so registration adopts it + // instead of tripping the rotation continuity check. + state.broker.Status.BrokerID = nil + return ctrl.Result{Requeue: true}, nil + } + state.phase = redpandav1alpha2.BrokerPhaseStuck + return ctrl.Result{RequeueAfter: periodicRequeue}, nil +} + +func (r *BrokerReconciler) reconcilePodRotation(ctx context.Context, state *brokerReconciliationState, cluster cluster.Cluster) (ctrl.Result, error) { + if state.pod == nil { + return ctrl.Result{}, nil + } + broker := state.broker + // PodOutdated covers both the config checksum and the restart-requiring + // cluster-config version (see resources.MarkBrokersForRestart); the + // recreated pod inherits both from the desired template, clearing the + // drift. + if !broker.PodOutdated(state.pod) { + return ctrl.Result{}, nil + } + + l := log.FromContext(ctx) + if !state.granted { + l.Info("pod needs rotation but no roll-grant", "name", state.pod.Name) + state.phase = redpandav1alpha2.BrokerPhaseRunning + return ctrl.Result{RequeueAfter: periodicRequeue}, nil + } + if broker.Status.BrokerID != nil { + drained, err := r.ensureDrained(ctx, broker) + if err != nil { + return ctrl.Result{}, fmt.Errorf("draining broker %d: %w", *broker.Status.BrokerID, err) + } + if !drained { + // Re-check quickly: while this broker holds the roll-grant, no + // other broker can roll — every second spent waiting here extends + // the whole fleet's roll duration. + l.Info("waiting for leadership drain before rotation", "brokerID", *broker.Status.BrokerID) + state.phase = redpandav1alpha2.BrokerPhaseProvisioning + return ctrl.Result{RequeueAfter: requeueDrain}, nil + } + } + l.Info("rotating pod", "name", state.pod.Name, + "oldChecksum", state.pod.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation], + "newChecksum", broker.Spec.PodTemplate.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation]) + if err := cluster.GetClient().Delete(ctx, state.pod); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil +} + +// reconcileBrokerRegistration verifies cluster membership on EVERY pass with +// a ready pod: it resolves the node_id the pod currently reports and checks +// registration continuity (RFC rolling step 3: after a rotation the broker +// must come back with the SAME node_id). The BrokerRegistered condition is +// recomputed from this pass's observation, so it can never report stale +// pre-rotation state, and the cluster controller only revokes a roll-grant +// once continuity has been re-verified. +func (r *BrokerReconciler) reconcileBrokerRegistration(ctx context.Context, state *brokerReconciliationState, _ cluster.Cluster) (ctrl.Result, error) { + broker := state.broker + // A decommissioning broker is leaving the cluster: there is no + // registration to maintain, and blocking the chain here would prevent + // reconcileDecommission from ever observing completion (the broker + // disappears from membership when the decommission finishes). + if broker.Spec.Decommission { + return ctrl.Result{}, nil + } + if state.pod == nil || !isPodReady(state.pod) { + return ctrl.Result{}, nil + } + + l := log.FromContext(ctx) + podName := broker.PodName() + + currentID, err := r.resolveBrokerID(ctx, broker, podName) + if err != nil { + l.Info("could not resolve broker ID, will retry", "error", err) + return ctrl.Result{RequeueAfter: requeueShort}, nil + } + if currentID == nil { + if broker.Status.BrokerID != nil { + // Was registered, no longer a member (e.g. removed out of + // band): report unverified but let the chain continue. + l.Info("broker no longer present in cluster membership", "brokerID", *broker.Status.BrokerID) + return ctrl.Result{}, nil + } + // Pod is up but not yet a cluster member. + return ctrl.Result{RequeueAfter: requeueShort}, nil + } + + if broker.Status.BrokerID == nil { + broker.Status.BrokerID = currentID + } + if *currentID != *broker.Status.BrokerID { + // The pod re-registered with a fresh identity — its data dir did not + // survive. Refuse to adopt the new ID and surface the conflict; this + // needs an operator decision (replace the broker), not silent + // acceptance. + state.registrationConflict = fmt.Sprintf( + "broker re-registered with node_id %d, expected %d", *currentID, *broker.Status.BrokerID) + state.phase = redpandav1alpha2.BrokerPhaseStuck + l.Error(fmt.Errorf("node_id changed from %d to %d", *broker.Status.BrokerID, *currentID), + "broker identity changed — not disabling maintenance mode") + return ctrl.Result{RequeueAfter: periodicRequeue}, nil + } + + state.registrationVerified = true + if err := r.disableMaintenanceMode(ctx, broker); err != nil { + l.Info("could not disable maintenance mode", "error", err) + } + + return ctrl.Result{}, nil +} + +func (r *BrokerReconciler) reconcileDecommission(ctx context.Context, state *brokerReconciliationState, cluster cluster.Cluster) (ctrl.Result, error) { + broker := state.broker + if !broker.Spec.Decommission { + // Unsetting the intent field mid-decommission recommissions the + // broker via the admin API (RFC Q2 review decision). + if broker.Status.Phase == redpandav1alpha2.BrokerPhaseDecommissioning && broker.Status.BrokerID != nil { + return r.executeRecommission(ctx, state) + } + return ctrl.Result{}, nil + } + + if broker.Status.BrokerID == nil { + // Decommissioned is terminal: the identity was removed and BrokerID + // cleared on completion — don't relabel it as in-progress. + if broker.Status.Phase == redpandav1alpha2.BrokerPhaseDecommissioned { + state.phase = redpandav1alpha2.BrokerPhaseDecommissioned + return ctrl.Result{}, nil + } + state.phase = redpandav1alpha2.BrokerPhaseDecommissioning + return ctrl.Result{}, nil + } + + decommResult, err := r.executeDecommission(ctx, broker) + if err != nil { + return ctrl.Result{}, err + } + state.phase = decommResult.phase + if decommResult.requeue { + return ctrl.Result{RequeueAfter: periodicRequeue}, nil + } + + if state.phase == redpandav1alpha2.BrokerPhaseDecommissioned { + // The identity is gone from the cluster for good. Clearing it lets + // a later revival (intent unset after completion) adopt the fresh + // node_id instead of tripping the rotation continuity check. + broker.Status.BrokerID = nil + l := log.FromContext(ctx) + k8sClient := cluster.GetClient() + podName := broker.PodName() + + if state.pod != nil { + l.Info("deleting pod after decommission", "name", podName) + if err := k8sClient.Delete(ctx, state.pod); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + for _, vct := range broker.Spec.Storage.VolumeClaimTemplates { + pvcName := fmt.Sprintf("%s-%s", vct.Name, podName) + var pvc corev1.PersistentVolumeClaim + if err := k8sClient.Get(ctx, client.ObjectKey{Name: pvcName, Namespace: broker.Namespace}, &pvc); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + continue + } + l.Info("deleting PVC after decommission", "name", pvcName) + if err := k8sClient.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + for _, ec := range broker.Spec.Storage.ExistingClaims { + var pvc corev1.PersistentVolumeClaim + if err := k8sClient.Get(ctx, client.ObjectKey{Name: ec.Name, Namespace: broker.Namespace}, &pvc); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + continue + } + l.Info("deleting PVC after decommission", "name", ec.Name) + if err := k8sClient.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + } + + return ctrl.Result{}, nil +} + +func (r *BrokerReconciler) syncBrokerStatus(ctx context.Context, state *brokerReconciliationState, k8sCluster cluster.Cluster, result ctrl.Result) (ctrl.Result, error) { + broker := state.broker + pod := state.pod + if pod == nil { + pod = &corev1.Pod{} + } + k8sClient := k8sCluster.GetClient() + status := statuses.NewBroker() + + phase := state.phase + if phase == "" { + phase = redpandav1alpha2.BrokerPhaseProvisioning + if isPodReady(pod) { + phase = redpandav1alpha2.BrokerPhaseRunning + } + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodScheduled && cond.Status == corev1.ConditionFalse && cond.Reason == "Unschedulable" { + phase = redpandav1alpha2.BrokerPhaseStuck + } + } + if reason := podStuckReason(pod); reason != "" { + phase = redpandav1alpha2.BrokerPhaseStuck + } + } + if state.registrationConflict != "" { + phase = redpandav1alpha2.BrokerPhaseStuck + } + + broker.Status.Phase = phase + broker.Status.PodName = pod.Name + broker.Status.PodIP = pod.Status.PodIP + + if isPodReady(pod) { + status.SetReady(statuses.BrokerReadyReasonReady) + } else { + status.SetReady(statuses.BrokerReadyReasonNotReady, "Pod is not ready") + } + + switch state.pod { + case nil: + status.SetPodScheduled(statuses.BrokerPodScheduledReasonPodMissing, "Pod not found") + default: + scheduledMessage := "Pod is not scheduled" + scheduled := false + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodScheduled { + scheduled = c.Status == corev1.ConditionTrue + if c.Message != "" { + scheduledMessage = c.Message + } + break + } + } + if scheduled { + status.SetPodScheduled(statuses.BrokerPodScheduledReasonScheduled) + } else { + status.SetPodScheduled(statuses.BrokerPodScheduledReasonUnschedulable, scheduledMessage) + } + } + + // BrokerRegistered mirrors THIS pass's admin-API observation, so it can + // never report stale pre-rotation registration (RFC rolling step 3: a + // rotated broker must come back with the same node_id before the roll is + // considered complete). + switch { + case state.registrationConflict != "": + status.SetBrokerRegistered(statuses.BrokerBrokerRegisteredReasonIdentityChanged, state.registrationConflict) + case state.registrationVerified: + status.SetBrokerRegistered(statuses.BrokerBrokerRegisteredReasonRegistered, fmt.Sprintf("Broker ID %d", *broker.Status.BrokerID)) + default: + status.SetBrokerRegistered(statuses.BrokerBrokerRegisteredReasonNotRegistered, "Broker registration not verified this reconcile") + } + + if broker.PodOutdated(pod) { + status.SetConfigSynced(statuses.BrokerConfigSyncedReasonOutdated, + fmt.Sprintf("desired checksum=%s version=%s, pod checksum=%s version=%s", + broker.Spec.PodTemplate.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation], + broker.Spec.PodTemplate.Annotations[redpandav1alpha2.BrokerClusterConfigVersionAnnotation], + pod.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation], + pod.Annotations[redpandav1alpha2.BrokerClusterConfigVersionAnnotation])) + } else { + status.SetConfigSynced(statuses.BrokerConfigSyncedReasonSynced) + } + + allBound := true + for _, vct := range broker.Spec.Storage.VolumeClaimTemplates { + pvcName := fmt.Sprintf("%s-%s", vct.Name, broker.PodName()) + var pvc corev1.PersistentVolumeClaim + if err := k8sClient.Get(ctx, client.ObjectKey{Name: pvcName, Namespace: broker.Namespace}, &pvc); err != nil || pvc.Status.Phase != corev1.ClaimBound { + allBound = false + break + } + } + for _, ec := range broker.Spec.Storage.ExistingClaims { + var pvc corev1.PersistentVolumeClaim + if err := k8sClient.Get(ctx, client.ObjectKey{Name: ec.Name, Namespace: broker.Namespace}, &pvc); err != nil || pvc.Status.Phase != corev1.ClaimBound { + allBound = false + break + } + } + if allBound { + status.SetStorageBound(statuses.BrokerStorageBoundReasonBound) + } else { + status.SetStorageBound(statuses.BrokerStorageBoundReasonPending, "One or more PVCs are not bound") + } + + // Quiesced and Stable are derived by the generated roll-up: Quiesced when + // every condition above was evaluated without transient errors, Stable + // when Ready, StorageBound, BrokerRegistered, ConfigSynced, and Quiesced + // are all True (statuses.yaml rollup). + // + // Skip the API write when nothing changed (RFC Q11): reconciles fire on + // every pod/PVC event plus a periodic requeue, and unconditional status + // PUTs across N brokers add up. + conditionsChanged := status.UpdateConditions(broker) + initial := state.initialStatus + fieldsChanged := initial == nil || + initial.Phase != broker.Status.Phase || + initial.PodName != broker.Status.PodName || + initial.PodIP != broker.Status.PodIP || + !ptr.Equal(initial.BrokerID, broker.Status.BrokerID) + if conditionsChanged || fieldsChanged { + if err := k8sClient.Status().Update(ctx, broker); err != nil { + return ctrl.Result{}, err + } + } + if !result.IsZero() { + return result, nil + } + return ctrl.Result{RequeueAfter: periodicRequeue}, nil +} + +// executeRecommission cancels an in-flight decommission after the intent +// field was unset. Recommission is only possible while the decommission is +// still in progress; if it already completed (or the broker is unknown), the +// phase simply falls back to being computed from pod state and the standard +// pod-ensure path revives the broker slot. +func (r *BrokerReconciler) executeRecommission(ctx context.Context, state *brokerReconciliationState) (ctrl.Result, error) { + l := log.FromContext(ctx) + broker := state.broker + brokerID := int(*broker.Status.BrokerID) + + admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) + if err != nil { + return ctrl.Result{}, err + } + defer admin.Close() + + l.Info("recommissioning broker, decommission intent was removed", "brokerID", brokerID) + if err := admin.RecommissionBroker(ctx, brokerID); err != nil { + // Not decommissioning (never started, or already finished): nothing + // to cancel — let the phase recompute from pod state. + if strings.Contains(err.Error(), "is not decommissioning") { + l.Info("broker is not decommissioning, nothing to recommission", "brokerID", brokerID) + } else { + return ctrl.Result{}, fmt.Errorf("recommissioning broker %d: %w", brokerID, err) + } + } + state.phase = "" // recompute from pod state + return ctrl.Result{}, nil +} + +type decommissionResult struct { + phase redpandav1alpha2.BrokerPhase + requeue bool +} + +func (r *BrokerReconciler) executeDecommission(ctx context.Context, broker *redpandav1alpha2.Broker) (decommissionResult, error) { + l := log.FromContext(ctx) + brokerID := int(*broker.Status.BrokerID) + + admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) + if err != nil { + return decommissionResult{phase: redpandav1alpha2.BrokerPhaseDecommissioning}, err + } + defer admin.Close() + + // Last-broker guard (RFC Q2). + brokers, err := admin.Brokers(ctx) + if err != nil { + return decommissionResult{phase: redpandav1alpha2.BrokerPhaseDecommissioning}, err + } + if len(brokers) <= 1 { + l.Info("blocking decommission: last broker in cluster", "brokerID", brokerID) + return decommissionResult{phase: redpandav1alpha2.BrokerPhaseStuck}, nil + } + + status, err := admin.DecommissionBrokerStatus(ctx, brokerID) + if err != nil { + if strings.Contains(err.Error(), "is not decommissioning") { + l.Info("initiating decommission", "brokerID", brokerID) + if err := admin.DecommissionBroker(ctx, brokerID); err != nil { + return decommissionResult{phase: redpandav1alpha2.BrokerPhaseDecommissioning}, err + } + return decommissionResult{phase: redpandav1alpha2.BrokerPhaseDecommissioning, requeue: true}, nil + } + return decommissionResult{phase: redpandav1alpha2.BrokerPhaseDecommissioning}, err + } + + if !status.Finished { + l.Info("decommission in progress", "brokerID", brokerID) + return decommissionResult{phase: redpandav1alpha2.BrokerPhaseDecommissioning, requeue: true}, nil + } + + l.Info("decommission finished", "brokerID", brokerID) + return decommissionResult{phase: redpandav1alpha2.BrokerPhaseDecommissioned}, nil +} + +// reconcileDelete handles Broker CR deletion (RFC Q2): +// +// - Decommission runs ONLY when Spec.Decommission is set — never on raw CR +// deletion alone — and a Stuck result (e.g. last-broker guard) blocks +// deletion instead of falling through to pod removal. +// - Raw deletion while the owning cluster is alive RELEASES the pod and +// PVCs (ownerRefs stripped): the broker keeps running, data survives, and +// the cluster controller recreates a Broker CR that re-adopts the pod — +// accidental deletion self-heals without a restart. +// - When the owning cluster itself is being deleted, the propagated +// deletion policy decides: "cascade" (default) lets the GC delete pod and +// PVCs with the CR — whole-cluster teardown, decommission is pointless — +// while "orphan" releases them. +func (r *BrokerReconciler) reconcileDelete(ctx context.Context, l logr.Logger, k8sClient client.Client, broker *redpandav1alpha2.Broker, podName string) (ctrl.Result, error) { + if !controllerutil.ContainsFinalizer(broker, brokerFinalizerName) { + return ctrl.Result{}, nil + } + + var pod corev1.Pod + err := k8sClient.Get(ctx, client.ObjectKey{Name: podName, Namespace: broker.Namespace}, &pod) + switch { + case apierrors.IsNotFound(err): + l.Info("pod already gone, removing finalizer") + + case err != nil: + return ctrl.Result{}, err + + case !metav1.IsControlledBy(&pod, broker): + l.Info("pod not owned by this Broker CR, removing finalizer (rollback case)", "owner", metav1.GetControllerOf(&pod)) + + case broker.Spec.Decommission: + // Deletion WITH explicit intent: decommission, then let the CR + // deletion cascade pod and PVCs. + // + // Status.BrokerID may be nil because a status update raced — resolve + // it live rather than skipping the decommission and leaving a dead + // membership entry behind. + if broker.Status.BrokerID == nil { + brokerID, err := r.resolveBrokerID(ctx, broker, podName) + if err != nil { + l.Info("could not resolve broker ID before decommission, will retry", "error", err) + return ctrl.Result{RequeueAfter: requeueShort}, nil + } + broker.Status.BrokerID = brokerID + } + if broker.Status.BrokerID != nil { + result, err := r.executeDecommission(ctx, broker) + if err != nil { + return ctrl.Result{}, err + } + broker.Status.Phase = result.phase + if err := k8sClient.Status().Update(ctx, broker); err != nil { + return ctrl.Result{}, err + } + if result.phase == redpandav1alpha2.BrokerPhaseStuck { + l.Info("decommission blocked, holding deletion", "name", broker.Name) + return ctrl.Result{RequeueAfter: periodicRequeue}, nil + } + if result.requeue { + return ctrl.Result{RequeueAfter: periodicRequeue}, nil + } + } + l.Info("deleting pod after decommission", "name", podName) + if err := k8sClient.Delete(ctx, &pod); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + + default: + // Deletion WITHOUT intent: never decommission (RFC Q2). + if r.deletionPolicy(ctx, l, k8sClient, broker) == deletionPolicyCascade { + // Owning cluster is being torn down with the default cascade + // policy: leave pod and PVCs to the GC (owned by this CR). + l.Info("cluster teardown with cascade policy, removing finalizer", "name", broker.Name) + } else { + if err := r.releaseBrokerResources(ctx, l, k8sClient, broker, &pod); err != nil { + return ctrl.Result{}, err + } + } + } + + controllerutil.RemoveFinalizer(broker, brokerFinalizerName) + if err := k8sClient.Update(ctx, broker); err != nil { + return ctrl.Result{}, err + } + l.Info("removed finalizer, Broker CR will be deleted") return ctrl.Result{}, nil } + +type brokerDeletionPolicy string + +const ( + deletionPolicyCascade brokerDeletionPolicy = "cascade" + deletionPolicyOrphan brokerDeletionPolicy = "orphan" +) + +// deletionPolicy decides what happens to the pod and PVCs when a Broker CR is +// deleted without decommission intent. While the owning cluster is alive the +// answer is always orphan (release + self-heal); during cluster teardown the +// policy propagated onto the Broker (feature.BrokerDeletionPolicy, default +// cascade) decides. On any uncertainty the answer is orphan — releasing a pod +// is recoverable, destroying its data is not. +func (r *BrokerReconciler) deletionPolicy(ctx context.Context, l logr.Logger, k8sClient client.Client, broker *redpandav1alpha2.Broker) brokerDeletionPolicy { + owner := metav1.GetControllerOf(broker) + if owner == nil { + // Unowned Broker deleted directly: release rather than destroy. + return deletionPolicyOrphan + } + + var ownerObj client.Object + switch { + case owner.Kind == "Cluster" && strings.HasPrefix(owner.APIVersion, "redpanda.vectorized.io/"): + ownerObj = &vectorizedv1alpha1.Cluster{} + case owner.Kind == "Redpanda" && strings.HasPrefix(owner.APIVersion, "cluster.redpanda.com/"): + ownerObj = &redpandav1alpha2.Redpanda{} + default: + return deletionPolicyOrphan + } + + ownerDeleting := false + err := k8sClient.Get(ctx, client.ObjectKey{Name: owner.Name, Namespace: broker.Namespace}, ownerObj) + switch { + case err == nil: + ownerDeleting = !ownerObj.GetDeletionTimestamp().IsZero() + case apierrors.IsNotFound(err): + ownerDeleting = true + default: + l.Info("could not determine owner state, releasing rather than destroying", "owner", owner.Name, "error", err) + } + if !ownerDeleting { + return deletionPolicyOrphan + } + + if feature.BrokerDeletionPolicy.Get(ctx, broker) == string(deletionPolicyOrphan) { + return deletionPolicyOrphan + } + return deletionPolicyCascade +} + +// releaseBrokerResources strips this Broker's ownerRefs from its pod and PVCs +// so they survive the CR deletion. +func (r *BrokerReconciler) releaseBrokerResources(ctx context.Context, l logr.Logger, k8sClient client.Client, broker *redpandav1alpha2.Broker, pod *corev1.Pod) error { + if removeOwnerRefByUID(pod, broker.UID) { + l.Info("releasing pod from deleted Broker CR", "pod", pod.Name) + if err := k8sClient.Update(ctx, pod); err != nil { + return fmt.Errorf("releasing pod %s: %w", pod.Name, err) + } + } + + var pvcNames []string + for _, vct := range broker.Spec.Storage.VolumeClaimTemplates { + pvcNames = append(pvcNames, fmt.Sprintf("%s-%s", vct.Name, broker.PodName())) + } + for _, ec := range broker.Spec.Storage.ExistingClaims { + pvcNames = append(pvcNames, ec.Name) + } + for _, name := range pvcNames { + var pvc corev1.PersistentVolumeClaim + if err := k8sClient.Get(ctx, client.ObjectKey{Name: name, Namespace: broker.Namespace}, &pvc); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return err + } + if removeOwnerRefByUID(&pvc, broker.UID) { + l.Info("releasing PVC from deleted Broker CR", "pvc", name) + if err := k8sClient.Update(ctx, &pvc); err != nil { + return fmt.Errorf("releasing PVC %s: %w", name, err) + } + } + } + return nil +} + +func removeOwnerRefByUID(obj client.Object, uid types.UID) bool { + refs := obj.GetOwnerReferences() + for i, ref := range refs { + if ref.UID == uid { + obj.SetOwnerReferences(append(refs[:i], refs[i+1:]...)) + return true + } + } + return false +} + +// ensureDrained enables maintenance mode and returns true when leadership drain +// is complete. Callers should requeue until this returns true. +func (r *BrokerReconciler) ensureDrained(ctx context.Context, broker *redpandav1alpha2.Broker) (bool, error) { + l := log.FromContext(ctx) + brokerID := int(*broker.Status.BrokerID) + + admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) + if err != nil { + return false, err + } + defer admin.Close() + + brokers, err := admin.Brokers(ctx) + if err != nil { + return false, err + } + for _, b := range brokers { + if b.NodeID != brokerID { + continue + } + if b.Maintenance != nil && b.Maintenance.Finished != nil && *b.Maintenance.Finished { + l.Info("leadership drain complete", "brokerID", brokerID) + return true, nil + } + if b.Maintenance != nil && b.Maintenance.Draining { + l.Info("leadership drain in progress", "brokerID", brokerID, + "partitions", b.Maintenance.Partitions, "transferring", b.Maintenance.Transferring) + return false, nil + } + // Not yet in maintenance mode — enable it. + if err := admin.EnableMaintenanceMode(ctx, brokerID); err != nil { + return false, err + } + l.Info("enabled maintenance mode", "brokerID", brokerID) + return false, nil + } + // Broker not found in admin API — skip drain, let rotation proceed. + return true, nil +} + +// remediatePVAffinity handles the dead-node recovery path: when a pod +// can't schedule because its PVCs are bound to PVs pinned to a node +// that no longer exists, we delete the affected PVCs and pod so the +// next reconcile recreates everything on a live node. +// +// PV identification and Retain-patching is delegated to +// [pvcunbinder.DeadNodePVCs]. Only VolumeClaimTemplate PVCs are +// remediated — ExistingClaims are externally-managed and the controller +// doesn't own their spec, so those are left for the admin to handle. +// +// Returns true if remediation was performed (caller should requeue). +// Returns false if the timeout hasn't elapsed or the node still exists. +func (r *BrokerReconciler) remediatePVAffinity(ctx context.Context, l logr.Logger, k8sClient client.Client, apiReader client.Reader, broker *redpandav1alpha2.Broker, pod *corev1.Pod) (bool, error) { + if r.UnbindPVCsAfter <= 0 { + l.Info("PV affinity remediation disabled (unbind-pvcs-after=0)") + return false, nil + } + + // Timeout: only act after the pod has been stuck long enough. + for _, cond := range pod.Status.Conditions { + if cond.Type != corev1.PodScheduled || cond.Status != corev1.ConditionFalse { + continue + } + if delta := r.UnbindPVCsAfter - time.Since(cond.LastTransitionTime.Time); delta > 0 { + l.Info("PV affinity stuck but timeout not reached", "remaining", delta.Round(time.Second)) + return false, nil + } + } + + affectedPVCs, err := pvcunbinder.DeadNodePVCs(ctx, k8sClient, apiReader, pod) + if err != nil { + return false, err + } + if len(affectedPVCs) == 0 { + return false, nil + } + + // Skip ExistingClaims — the controller doesn't own their spec. + existingClaimNames := map[string]bool{} + for _, ec := range broker.Spec.Storage.ExistingClaims { + existingClaimNames[ec.Name] = true + } + var remediable []corev1.PersistentVolumeClaim + for _, pvc := range affectedPVCs { + if existingClaimNames[pvc.Name] { + l.Info("ExistingClaim PVC stuck on dead node, skipping (admin must handle)", "pvc", pvc.Name) + continue + } + remediable = append(remediable, pvc) + } + if len(remediable) == 0 { + return false, nil + } + + for i := range remediable { + l.Info("deleting PVC pinned to dead node", "pvc", remediable[i].Name) + if err := k8sClient.Delete(ctx, &remediable[i]); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("deleting PVC %s: %w", remediable[i].Name, err) + } + } + + l.Info("deleting pod for PV affinity remediation", "pod", pod.Name) + if err := k8sClient.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("deleting pod %s: %w", pod.Name, err) + } + + return true, nil +} + +func (r *BrokerReconciler) disableMaintenanceMode(ctx context.Context, broker *redpandav1alpha2.Broker) error { + admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) + if err != nil { + return err + } + defer admin.Close() + return admin.DisableMaintenanceMode(ctx, int(*broker.Status.BrokerID), false) +} + +func (r *BrokerReconciler) resolveBrokerID(ctx context.Context, broker *redpandav1alpha2.Broker, podName string) (*int32, error) { + admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) + if err != nil { + return nil, err + } + defer admin.Close() + + brokers, err := admin.Brokers(ctx) + if err != nil { + return nil, err + } + for _, b := range brokers { + if strings.Split(b.InternalRPCAddress, ".")[0] == podName { + return ptr.To(int32(b.NodeID)), nil + } + } + return nil, nil +} + +// hasValidRollGrant returns true if the Broker CR carries a roll-grant +// annotation whose config-checksum portion matches the Broker's desired +// pod template checksum and whose deadline has not passed. +func hasValidRollGrant(ctx context.Context, broker *redpandav1alpha2.Broker) bool { + grant := feature.RollGrant.Get(ctx, broker) + if grant == "" { + return false + } + grantChecksum, deadline, ok := feature.ParseRollGrant(grant) + if !ok { + return false + } + if grantChecksum != broker.Spec.PodTemplate.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation] { + return false + } + return time.Now().Before(deadline) +} + +// podStuckReason reports the container waiting reason when a pod is wedged +// in a state kubelet will not recover on its own (crash-looping, unable to +// pull its image, or failing container creation). +func podStuckReason(pod *corev1.Pod) string { + statuses := make([]corev1.ContainerStatus, 0, len(pod.Status.ContainerStatuses)+len(pod.Status.InitContainerStatuses)) + statuses = append(statuses, pod.Status.InitContainerStatuses...) + statuses = append(statuses, pod.Status.ContainerStatuses...) + for _, cs := range statuses { + if w := cs.State.Waiting; w != nil { + switch w.Reason { + case "CrashLoopBackOff", "ImagePullBackOff", "ErrImagePull", + "InvalidImageName", "CreateContainerConfigError", "CreateContainerError": + return w.Reason + } + } + } + return "" +} + +func isPodReady(pod *corev1.Pod) bool { + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + return true + } + } + return false +} diff --git a/operator/internal/controller/redpanda/nodepool_controller.go b/operator/internal/controller/redpanda/nodepool_controller.go index 25e01d2b33..8747570c97 100644 --- a/operator/internal/controller/redpanda/nodepool_controller.go +++ b/operator/internal/controller/redpanda/nodepool_controller.go @@ -25,6 +25,7 @@ import ( "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlcontroller "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" @@ -63,12 +64,13 @@ func createCanonicalClusterNameList(mgr multicluster.Manager) []string { // SetupWithManager sets up the controller with the Manager. func (r *NodePoolReconciler) SetupWithManager(ctx context.Context, mgr multicluster.Manager, namespace string) error { - builder := mcbuilder.ControllerManagedBy(mgr). - For( - &redpandav1alpha2.NodePool{}, - mcbuilder.WithEngageWithLocalCluster(true), - mcbuilder.WithEngageWithProviderClusters(true), - ). + builder := mcbuilder.ControllerManagedBy(mgr).WithOptions(ctrlcontroller.TypedOptions[mcreconcile.Request]{ + SkipNameValidation: ptr.To(true), + }).For( + &redpandav1alpha2.NodePool{}, + mcbuilder.WithEngageWithLocalCluster(true), + mcbuilder.WithEngageWithProviderClusters(true), + ). Watches(&appsv1.StatefulSet{}, mchandler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { labels := o.GetLabels() if labels == nil { diff --git a/operator/internal/controller/redpanda/redpanda_controller.go b/operator/internal/controller/redpanda/redpanda_controller.go index 7c95c5c33b..c980697dc4 100644 --- a/operator/internal/controller/redpanda/redpanda_controller.go +++ b/operator/internal/controller/redpanda/redpanda_controller.go @@ -35,6 +35,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" + ctrlcontroller "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" @@ -151,7 +152,9 @@ type RedpandaReconciler struct { // SetupWithManager sets up the controller with the Manager. func (r *RedpandaReconciler) SetupWithManager(ctx context.Context, mgr multicluster.Manager, namespace string) error { - builder := mcbuilder.ControllerManagedBy(mgr) + builder := mcbuilder.ControllerManagedBy(mgr).WithOptions(ctrlcontroller.TypedOptions[mcreconcile.Request]{ + SkipNameValidation: ptr.To(true), + }) if err := r.LifecycleClient.WatchResources(builder, &redpandav1alpha2.Redpanda{}, mgr.GetClusterNames()); err != nil { return err diff --git a/operator/internal/controller/redpanda/testdata/role.yaml b/operator/internal/controller/redpanda/testdata/role.yaml index 1af17c9560..88d88e09f6 100644 --- a/operator/internal/controller/redpanda/testdata/role.yaml +++ b/operator/internal/controller/redpanda/testdata/role.yaml @@ -9,6 +9,7 @@ rules: resources: - configmaps - endpoints + - persistentvolumeclaims - pods - secrets - serviceaccounts @@ -42,17 +43,6 @@ rules: - nodes verbs: - get -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - delete - - get - - list - - patch - - update - - watch - apiGroups: - "" resources: @@ -313,6 +303,14 @@ rules: - patch - update - watch +- apiGroups: + - redpanda.vectorized.io + resources: + - clusters + verbs: + - get + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role From 68efddcb12fc2824c5a8df17dc660a12ca11bf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Thu, 9 Jul 2026 12:06:44 +0200 Subject: [PATCH 03/29] operator: add Broker controller integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers the full lifecycle against a real Redpanda cluster on k3d: pod adoption from an orphan-deleted StatefulSet, ungated pod creation, grant-gated rotation (and refusal to rotate without a grant), spec-driven decommission to completion with pod/PVC cleanup, decommission-on-delete with explicit intent, raw deletion releasing pod and PVCs, slot revival after clearing a completed decommission, the last-broker guard, finalizer behavior for gone/foreign pods, and dead-node PV-affinity remediation (scoped to agent nodes — deleting the k3d server would take down the API server for the whole suite). testenv exposes the underlying k3d cluster for the node-manipulation tests. Co-Authored-By: Claude Fable 5 --- .../redpanda/broker_controller_test.go | 900 ++++++++++++++++++ operator/internal/testenv/testenv.go | 4 + 2 files changed, 904 insertions(+) create mode 100644 operator/internal/controller/redpanda/broker_controller_test.go diff --git a/operator/internal/controller/redpanda/broker_controller_test.go b/operator/internal/controller/redpanda/broker_controller_test.go new file mode 100644 index 0000000000..8ed0b98da1 --- /dev/null +++ b/operator/internal/controller/redpanda/broker_controller_test.go @@ -0,0 +1,900 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package redpanda_test + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/redpanda-data/common-go/kube" + "github.com/redpanda-data/common-go/otelutil/log" + "github.com/redpanda-data/common-go/otelutil/trace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/yaml" + + redpandav1alpha2 "github.com/redpanda-data/redpanda-operator/operator/api/redpanda/v1alpha2" + crds "github.com/redpanda-data/redpanda-operator/operator/config/crd/bases" + "github.com/redpanda-data/redpanda-operator/operator/internal/controller" + "github.com/redpanda-data/redpanda-operator/operator/internal/controller/redpanda" + "github.com/redpanda-data/redpanda-operator/operator/internal/lifecycle" + "github.com/redpanda-data/redpanda-operator/operator/internal/statuses" + "github.com/redpanda-data/redpanda-operator/operator/internal/testenv" + internalclient "github.com/redpanda-data/redpanda-operator/operator/pkg/client" + "github.com/redpanda-data/redpanda-operator/pkg/multicluster" + "github.com/redpanda-data/redpanda-operator/pkg/testutil" +) + +func TestIntegrationBrokerController(t *testing.T) { + testutil.SkipIfNotIntegration(t) + suite.Run(t, new(BrokerControllerSuite)) +} + +type BrokerControllerSuite struct { + suite.Suite + + env *testenv.Env + clientFactory internalclient.ClientFactory +} + +var _ suite.SetupAllSuite = (*BrokerControllerSuite)(nil) + +func (s *BrokerControllerSuite) setup() (*testing.T, context.Context, context.CancelFunc, client.Client) { + t := s.T() + t.Parallel() + return s.setupNamespace(t) +} + +// setupSerial is setup without t.Parallel, for tests that disrupt shared +// cluster infrastructure (e.g. deleting k3d nodes): running those alongside +// parallel tests strands the other tests' pods and PVCs on the dead node. +func (s *BrokerControllerSuite) setupSerial() (*testing.T, context.Context, context.CancelFunc, client.Client) { + return s.setupNamespace(s.T()) +} + +func (s *BrokerControllerSuite) setupNamespace(t *testing.T) (*testing.T, context.Context, context.CancelFunc, client.Client) { + ctx, cancel := context.WithTimeout(trace.Test(t), 15*time.Minute) + ns := s.env.CreateTestNamespace(t) + return t, ctx, cancel, ns.Client +} + +func (s *BrokerControllerSuite) SetupSuite() { + t := s.T() + ctx := trace.Test(t) + + importImages := []string{ + "localhost/redpanda-operator:dev", + "ghcr.io/loft-sh/vcluster-pro:0.35.1", + "registry.k8s.io/kube-controller-manager:v1.33.12", + "registry.k8s.io/kube-apiserver:v1.33.12", + "quay.io/jetstack/cert-manager-controller:v1.17.2", + "quay.io/jetstack/cert-manager-cainjector:v1.17.2", + "quay.io/jetstack/cert-manager-webhook:v1.17.2", + "coredns/coredns:1.11.1", + } + if repo := os.Getenv("TEST_REDPANDA_REPO"); repo != "" { + if version := os.Getenv("TEST_REDPANDA_VERSION"); version != "" { + importImages = append(importImages, fmt.Sprintf("%s:%s", repo, version)) + } + } + + s.env = testenv.New(t, testenv.Options{ + Scheme: controller.V2Scheme, + CRDs: crds.All(), + Logger: log.FromContext(ctx), + SkipVCluster: true, + WatchAllNamespaces: true, + ImportImages: importImages, + }) + + suiteClient := s.env.Client() + + s.env.SetupManager(s.setupRBAC(ctx, suiteClient), func(mgr multicluster.Manager) error { + dialer := kube.NewPodDialer(mgr.GetLocalManager().GetConfig()) + s.clientFactory = internalclient.NewFactory(mgr, nil).WithDialer(dialer.DialContext) + + require.NoError(t, (&redpanda.NodePoolReconciler{ + Manager: mgr, + }).SetupWithManager(ctx, mgr, "")) + + require.NoError(t, (&redpanda.RedpandaReconciler{ + Manager: mgr, + ClientFactory: s.clientFactory, + LifecycleClient: lifecycle.NewResourceClient(mgr, lifecycle.V2ResourceManagers( + lifecycle.Image{Repository: os.Getenv("TEST_REDPANDA_REPO"), Tag: os.Getenv("TEST_REDPANDA_VERSION")}, + lifecycle.Image{Repository: "localhost/redpanda-operator", Tag: "dev"}, + lifecycle.CloudSecretsFlags{CloudSecretsEnabled: false}, + )), + UseNodePools: true, + }).SetupWithManager(ctx, mgr, "")) + + return redpanda.SetupBrokerController(ctx, mgr, s.clientFactory, "", 60*time.Second) + }) +} + +func (s *BrokerControllerSuite) setupRBAC(ctx context.Context, c client.Client) string { + t := s.T() + + roles, err := kube.DecodeYAML(operatorRBAC, c.Scheme()) + require.NoError(t, err) + + role := roles[1].(*rbacv1.Role) + clusterRole := roles[0].(*rbacv1.ClusterRole) + + clusterRole.Rules = append(clusterRole.Rules, role.Rules...) + clusterRole.Rules = append(clusterRole.Rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"pods/portforward"}, + Verbs: []string{"*"}, + }, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list"}, + }) + + name := "testenv-" + testenv.RandString(6) + clusterRole.Name = name + + s.applyAndWait(t, ctx, c, clusterRole) + s.applyAndWait(t, ctx, c, + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + }, + &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Subjects: []rbacv1.Subject{ + {Kind: "ServiceAccount", Namespace: s.env.Namespace(), Name: name}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: clusterRole.Name, + }, + }, + ) + + return name +} + +// TestFinalizerPodGone verifies that deleting a Broker CR whose pod doesn't +// exist removes the finalizer and lets the CR be garbage-collected. +func (s *BrokerControllerSuite) TestFinalizerPodGone() { + t, ctx, cancel, c := s.setup() + defer cancel() + + broker := s.minimalBroker("no-such-cluster") + require.NoError(t, c.Create(ctx, broker)) + + s.waitForFinalizer(t, ctx, c, broker) + + require.NoError(t, c.Delete(ctx, broker)) + + s.waitForDeletion(t, ctx, c, broker) +} + +// TestFinalizerPodNotOwned verifies the rollback case: if the pod exists but is +// not owned by the Broker CR, the finalizer is removed without decommissioning, +// and the pod survives. +func (s *BrokerControllerSuite) TestFinalizerPodNotOwned() { + t, ctx, cancel, c := s.setup() + defer cancel() + + // The Broker's PodName() returns "-". + clusterName := "rollback-" + testenv.RandString(4) + podName := clusterName + "-0" + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "redpanda", + Image: "busybox", + Command: []string{"sleep", "3600"}, + }}, + }, + } + require.NoError(t, c.Create(ctx, pod)) + + broker := s.minimalBroker(clusterName) + require.NoError(t, c.Create(ctx, broker)) + + s.waitForFinalizer(t, ctx, c, broker) + + require.NoError(t, c.Delete(ctx, broker)) + + s.waitForDeletion(t, ctx, c, broker) + + // Pod must still exist. + var surviving corev1.Pod + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(pod), &surviving)) + require.Equal(t, podName, surviving.Name) +} + +// TestFinalizerDecommission verifies the decommission-on-delete path: with +// spec.decommission set, deleting the Broker CR decommissions the broker, +// deletes its pod, and removes the finalizer. +func (s *BrokerControllerSuite) TestFinalizerDecommission() { + t, ctx, cancel, c := s.setup() + defer cancel() + + rp, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ + grantDuration: 10 * time.Minute, + }) + + admin, err := s.clientFactory.RedpandaAdminClient(ctx, brokers[0]) + require.NoError(t, err) + + require.EventuallyWithT(t, func(ct *assert.CollectT) { + bs, err := admin.Brokers(ctx) + assert.NoError(ct, err) + assert.Len(ct, bs, 3) + }, 2*time.Minute, 5*time.Second) + + // Mark broker-2 for decommission, then delete its CR — deletion with + // intent decommissions before cleanup (RFC Q2). + target := brokers[2] + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + p := client.MergeFrom(target.DeepCopy()) + target.Spec.Decommission = true + require.NoError(t, c.Patch(ctx, target, p)) + require.NoError(t, c.Delete(ctx, target)) + + // The Broker CR should be fully deleted (finalizer removed after decommission). + s.waitForDeletion(t, ctx, c, target) + + // The pod should be gone (may take time for kubelet to terminate it). + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var deletedPod corev1.Pod + err := c.Get(ctx, client.ObjectKey{Name: fmt.Sprintf("%s-2", rp.Name), Namespace: rp.Namespace}, &deletedPod) + assert.True(ct, apierrors.IsNotFound(err), "expected pod to be deleted, got: %v", err) + }, 2*time.Minute, 5*time.Second) + + // Admin API should show 2 brokers. + require.EventuallyWithT(t, func(ct *assert.CollectT) { + bs, err := admin.Brokers(ctx) + assert.NoError(ct, err) + assert.Len(ct, bs, 2) + }, 5*time.Minute, 5*time.Second) + + admin.Close() +} + +// TestDecommissionIntentRemovedRevivesSlot verifies the recommission +// contract: clearing spec.decommission after the decommission completed +// revives the broker slot — the pod is recreated (ungated pod-ensure) and the +// broker rejoins the cluster. +func (s *BrokerControllerSuite) TestDecommissionIntentRemovedRevivesSlot() { + t, ctx, cancel, c := s.setup() + defer cancel() + + _, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ + grantDuration: 10 * time.Minute, + // Revival recreates PVCs from templates; ExistingClaims are + // adopt-only and were deleted by the completed decommission. + useVolumeClaimTemplates: true, + }) + + admin, err := s.clientFactory.RedpandaAdminClient(ctx, brokers[0]) + require.NoError(t, err) + defer admin.Close() + + require.EventuallyWithT(t, func(ct *assert.CollectT) { + bs, err := admin.Brokers(ctx) + assert.NoError(ct, err) + assert.Len(ct, bs, 3) + }, 2*time.Minute, 5*time.Second) + + // Decommission broker-2 to completion. + target := brokers[2] + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + p := client.MergeFrom(target.DeepCopy()) + target.Spec.Decommission = true + require.NoError(t, c.Patch(ctx, target, p)) + s.waitForPhase(t, ctx, c, target, redpandav1alpha2.BrokerPhaseDecommissioned) + + // Clear the intent: the slot revives with a fresh identity. + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + p = client.MergeFrom(target.DeepCopy()) + target.Spec.Decommission = false + require.NoError(t, c.Patch(ctx, target, p)) + + s.waitForPhase(t, ctx, c, target, redpandav1alpha2.BrokerPhaseRunning) + require.EventuallyWithT(t, func(ct *assert.CollectT) { + bs, err := admin.Brokers(ctx) + assert.NoError(ct, err) + assert.Len(ct, bs, 3) + }, 5*time.Minute, 5*time.Second) +} + +// TestFinalizerRawDeletionReleases verifies RFC Q2: deleting a Broker CR +// WITHOUT spec.decommission never decommissions — the pod and PVCs are +// released (ownerRefs stripped) and the broker keeps serving. +func (s *BrokerControllerSuite) TestFinalizerRawDeletionReleases() { + t, ctx, cancel, c := s.setup() + defer cancel() + + rp, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ + grantDuration: 10 * time.Minute, + }) + + admin, err := s.clientFactory.RedpandaAdminClient(ctx, brokers[0]) + require.NoError(t, err) + defer admin.Close() + + require.EventuallyWithT(t, func(ct *assert.CollectT) { + bs, err := admin.Brokers(ctx) + assert.NoError(ct, err) + assert.Len(ct, bs, 3) + }, 2*time.Minute, 5*time.Second) + + // Raw deletion: no decommission intent. + target := brokers[2] + targetPodName := fmt.Sprintf("%s-2", rp.Name) + require.NoError(t, c.Delete(ctx, target)) + + s.waitForDeletion(t, ctx, c, target) + + // The pod survives, released from the deleted CR. + var pod corev1.Pod + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: targetPodName, Namespace: rp.Namespace}, &pod)) + assert.Nil(t, metav1.GetControllerOf(&pod), "pod should have been released from the deleted Broker CR") + + // Membership is untouched: still 3 brokers. + bs, err := admin.Brokers(ctx) + require.NoError(t, err) + assert.Len(t, bs, 3) +} + +// TestSpecDecommission verifies the spec-driven decommission path: +// create a real Redpanda cluster, migrate pods to Broker CRs, then set +// spec.decommission=true on one Broker CR and verify it decommissions, +// deletes its pod and PVCs, and reaches Decommissioned phase. +func (s *BrokerControllerSuite) TestSpecDecommission() { + t, ctx, cancel, c := s.setup() + defer cancel() + + rp, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ + grantDuration: 10 * time.Minute, + }) + + admin, err := s.clientFactory.RedpandaAdminClient(ctx, brokers[0]) + require.NoError(t, err) + + require.EventuallyWithT(t, func(ct *assert.CollectT) { + bs, err := admin.Brokers(ctx) + assert.NoError(ct, err) + assert.Len(ct, bs, 3) + }, 2*time.Minute, 5*time.Second) + + // Set spec.decommission=true on broker-2. + target := brokers[2] + targetPodName := fmt.Sprintf("%s-2", rp.Name) + + // Record PVC names for later. + var targetPVCNames []string + for _, ec := range target.Spec.Storage.ExistingClaims { + targetPVCNames = append(targetPVCNames, ec.Name) + } + require.NotEmpty(t, targetPVCNames) + + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + p := client.MergeFrom(target.DeepCopy()) + target.Spec.Decommission = true + require.NoError(t, c.Patch(ctx, target, p)) + + // Wait for Decommissioned phase. + s.waitForPhase(t, ctx, c, target, redpandav1alpha2.BrokerPhaseDecommissioned) + + // The pod should be deleted. + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var deletedPod corev1.Pod + err := c.Get(ctx, client.ObjectKey{Name: targetPodName, Namespace: rp.Namespace}, &deletedPod) + assert.True(ct, apierrors.IsNotFound(err), "expected pod to be deleted, got: %v", err) + }, 2*time.Minute, 5*time.Second) + + // The PVCs should be deleted. + for _, pvcName := range targetPVCNames { + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var pvc corev1.PersistentVolumeClaim + err := c.Get(ctx, client.ObjectKey{Name: pvcName, Namespace: rp.Namespace}, &pvc) + assert.True(ct, apierrors.IsNotFound(err), "expected PVC %q to be deleted, got: %v", pvcName, err) + }, 2*time.Minute, 5*time.Second) + } + + // Admin API should show 2 brokers. + require.EventuallyWithT(t, func(ct *assert.CollectT) { + bs, err := admin.Brokers(ctx) + assert.NoError(ct, err) + assert.Len(ct, bs, 2) + }, 5*time.Minute, 5*time.Second) + + // The Broker CR should still exist (not deleted, just decommissioned). + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + assert.Equal(t, redpandav1alpha2.BrokerPhaseDecommissioned, target.Status.Phase) + + admin.Close() +} + +// TestPodCreatedWithoutGrant verifies that pod creation is NOT gated on a +// roll-grant (RFC Q5): pod-ensure is the unconditional first step, only +// disruptive actions (rotation, PV remediation) require a grant. +func (s *BrokerControllerSuite) TestPodCreatedWithoutGrant() { + t, ctx, cancel, c := s.setup() + defer cancel() + + broker := s.minimalBroker("no-such-cluster") + require.NoError(t, c.Create(ctx, broker)) + + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var pod corev1.Pod + assert.NoError(ct, c.Get(ctx, client.ObjectKey{Name: broker.PodName(), Namespace: broker.Namespace}, &pod)) + }, 2*time.Minute, time.Second, "pod should be created without a roll-grant") +} + +// TestPodRotationWithoutGrant verifies that when a broker's pod needs rotation +// (checksum mismatch) but no roll-grant is present, the broker stays in Running +// phase and sets ConfigSynced=False. +func (s *BrokerControllerSuite) TestPodRotationWithoutGrant() { + t, ctx, cancel, c := s.setup() + defer cancel() + + _, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ + grantDuration: 10 * time.Minute, + }) + + target := brokers[0] + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + + // Change the desired checksum so it no longer matches the pod's checksum + // (and no longer matches the existing roll-grant's checksum). + p := client.MergeFrom(target.DeepCopy()) + target.Spec.PodTemplate.Annotations["config.redpanda.com/checksum"] = "deliberately-wrong-checksum" + require.NoError(t, c.Patch(ctx, target, p)) + + // The broker should stay Running (not rotated) with ConfigSynced=False. + require.EventuallyWithT(t, func(ct *assert.CollectT) { + assert.NoError(ct, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + assert.Equal(ct, redpandav1alpha2.BrokerPhaseRunning, target.Status.Phase) + cond := apimeta.FindStatusCondition(target.Status.Conditions, statuses.BrokerConfigSynced) + if assert.NotNil(ct, cond, "ConfigSynced condition not found") { + assert.Equal(ct, metav1.ConditionFalse, cond.Status) + } + }, 2*time.Minute, 5*time.Second) +} + +// TestLastBrokerDecommissionGuard verifies that attempting to decommission the +// only broker in a cluster results in Stuck phase instead of proceeding. +func (s *BrokerControllerSuite) TestLastBrokerDecommissionGuard() { + t, ctx, cancel, c := s.setup() + defer cancel() + + _, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ + replicas: 1, + grantDuration: 10 * time.Minute, + }) + + target := brokers[0] + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + + p := client.MergeFrom(target.DeepCopy()) + target.Spec.Decommission = true + require.NoError(t, c.Patch(ctx, target, p)) + + s.waitForPhase(t, ctx, c, target, redpandav1alpha2.BrokerPhaseStuck) +} + +// TestPVAffinityRemediation verifies the dead-node recovery path: +// create a Redpanda cluster, migrate to Broker CRs, delete a k3d node, +// force-delete the stuck pod, and assert the Broker controller reports +// Stuck then — once granted a roll — remediates and recovers. +func (s *BrokerControllerSuite) TestPVAffinityRemediation() { + // Serial: this test deletes a shared k3d node; parallel neighbors would + // lose pods and node-pinned PVCs with it. + t, ctx, cancel, c := s.setupSerial() + defer cancel() + + _, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ + useVolumeClaimTemplates: true, + grantDuration: 30 * time.Minute, + }) + + // Pick a target broker whose pod runs on an AGENT node: k3d server nodes + // are schedulable, and deleting the server would take down the API + // server for the entire suite. + var target *redpandav1alpha2.Broker + var targetPod corev1.Pod + for _, b := range brokers { + var pod corev1.Pod + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: b.PodName(), Namespace: b.Namespace}, &pod)) + if pod.Spec.NodeName != "" && !strings.Contains(pod.Spec.NodeName, "server") { + target = b + targetPod = pod + break + } + } + require.NotNil(t, target, "no broker pod scheduled on an agent node") + targetNode := targetPod.Spec.NodeName + t.Logf("target pod %q is on node %q", targetPod.Name, targetNode) + + // Record the original PVC names for later comparison. + var originalPVCNames []string + for _, vol := range targetPod.Spec.Volumes { + if vol.PersistentVolumeClaim != nil { + originalPVCNames = append(originalPVCNames, vol.PersistentVolumeClaim.ClaimName) + } + } + require.NotEmpty(t, originalPVCNames, "target pod must have PVCs") + + // Record original PV names bound to these PVCs. + originalPVNames := map[string]string{} // pvcName -> pvName + for _, name := range originalPVCNames { + var pvc corev1.PersistentVolumeClaim + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: name, Namespace: target.Namespace}, &pvc)) + require.NotEmpty(t, pvc.Spec.VolumeName, "PVC %q must be bound", name) + originalPVNames[name] = pvc.Spec.VolumeName + } + + // Delete the k3d node. Ensure a replacement is always created, even on + // test failure, so other tests sharing this k3d cluster are not impacted. + var nodesBefore corev1.NodeList + require.NoError(t, s.env.Client().List(ctx, &nodesBefore)) + expectedNodeCount := len(nodesBefore.Items) + + t.Logf("deleting k3d node %q", targetNode) + require.NoError(t, s.env.Host().DeleteNode(targetNode)) + t.Cleanup(func() { + ctx := context.Background() + // Remove the stale Node object if the test failed before doing so + // itself: a dead node lingering in the API would both skew the count + // below and confuse later tests. + var staleNode corev1.Node + if err := s.env.Client().Get(ctx, client.ObjectKey{Name: targetNode}, &staleNode); err == nil { + if err := s.env.Client().Delete(ctx, &staleNode); err != nil { + t.Logf("WARNING: failed to delete stale Node object %q during cleanup: %v", targetNode, err) + } + } + var nodes corev1.NodeList + if err := s.env.Client().List(ctx, &nodes); err != nil { + t.Logf("WARNING: failed to list nodes during cleanup: %v", err) + return + } + ready := 0 + for _, node := range nodes.Items { + if node.Name == targetNode { + continue + } + for _, cond := range node.Status.Conditions { + if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue { + ready++ + break + } + } + } + if ready >= expectedNodeCount { + return + } + t.Logf("cluster has %d ready nodes (expected %d), creating replacement", ready, expectedNodeCount) + if err := s.env.Host().CreateNode(); err != nil { + t.Logf("WARNING: failed to create replacement k3d node during cleanup: %v", err) + } + }) + + // Delete the Kubernetes Node object so the Broker controller sees it as gone. + var nodeObj corev1.Node + require.EventuallyWithT(t, func(ct *assert.CollectT) { + // The node might take a moment to be reported as not-ready. + err := c.Get(ctx, client.ObjectKey{Name: targetNode}, &nodeObj) + if !assert.NoError(ct, err) { + return + } + assert.NoError(ct, c.Delete(ctx, &nodeObj)) + }, 1*time.Minute, 2*time.Second) + + // Force-delete the pod stuck in Terminating (kubelet on dead node can't confirm). + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var pod corev1.Pod + err := c.Get(ctx, client.ObjectKey{Name: targetPod.Name, Namespace: targetPod.Namespace}, &pod) + if apierrors.IsNotFound(err) { + return + } + assert.NoError(ct, err) + assert.NoError(ct, c.Delete(ctx, &pod, client.GracePeriodSeconds(0))) + }, 2*time.Minute, 2*time.Second) + + // Wait for Broker to report Stuck (pod recreated but can't schedule due to PV affinity). + t.Log("waiting for Broker to report Stuck") + s.waitForPhase(t, ctx, c, target, redpandav1alpha2.BrokerPhaseStuck) + + // Grant a fresh roll-grant for the remediation. + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + p := client.MergeFrom(target.DeepCopy()) + if target.Annotations == nil { + target.Annotations = map[string]string{} + } + checksum := target.Spec.PodTemplate.Annotations["config.redpanda.com/checksum"] + target.Annotations["operator.redpanda.com/roll-grant"] = checksum + "/" + strconv.FormatInt(time.Now().Add(30*time.Minute).Unix(), 10) + require.NoError(t, c.Patch(ctx, target, p)) + + // Create a replacement k3d node so the pod has somewhere to schedule. + t.Log("creating replacement k3d node") + require.NoError(t, s.env.Host().CreateNode()) + + // Wait for the broker to recover to Running. + t.Log("waiting for Broker to recover to Running") + s.waitForPhase(t, ctx, c, target, redpandav1alpha2.BrokerPhaseRunning) + + // Verify remediation happened: the old PVs should have Retain policy. + for pvcName, pvName := range originalPVNames { + var pv corev1.PersistentVolume + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: pvName}, &pv), + "original PV %q (from PVC %q) should still exist", pvName, pvcName) + assert.Equal(t, corev1.PersistentVolumeReclaimRetain, pv.Spec.PersistentVolumeReclaimPolicy, + "original PV %q should have been patched to Retain", pvName) + } + + // The target pod should now be on a different node. + var recoveredPod corev1.Pod + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: target.PodName(), Namespace: target.Namespace}, &recoveredPod)) + assert.NotEqual(t, targetNode, recoveredPod.Spec.NodeName, + "recovered pod should be on a different node") + t.Logf("recovered pod %q is now on node %q (was %q)", recoveredPod.Name, recoveredPod.Spec.NodeName, targetNode) +} + +// --- helpers --- + +type brokerClusterOpts struct { + replicas int + useVolumeClaimTemplates bool + grantDuration time.Duration +} + +// setupBrokerCluster creates a Redpanda cluster, orphan-deletes its +// StatefulSet, pauses reconciliation, creates Broker CRs for each pod, +// grants roll-grants, and waits for all brokers to reach Running. +// Defaults to 3 replicas if opts.replicas is 0. +func (s *BrokerControllerSuite) setupBrokerCluster(t *testing.T, ctx context.Context, c client.Client, opts brokerClusterOpts) (*redpandav1alpha2.Redpanda, []*redpandav1alpha2.Broker) { + replicas := opts.replicas + if replicas == 0 { + replicas = 3 + } + rp := s.minimalRP() + rp.Spec.ClusterSpec.Statefulset.Replicas = ptr.To(replicas) + s.applyAndWait(t, ctx, c, rp) + + // Pause reconciliation BEFORE orphan-deleting the StatefulSet: the + // Redpanda controller would otherwise recreate the STS, which then + // resurrects any pod the Broker controller later deletes (e.g. after a + // decommission), corrupting the test. + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(rp), rp)) + patch := client.MergeFrom(rp.DeepCopy()) + if rp.Annotations == nil { + rp.Annotations = map[string]string{} + } + rp.Annotations["cluster.redpanda.com/managed"] = "false" + require.NoError(t, c.Patch(ctx, rp, patch)) + + var found appsv1.StatefulSet + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: rp.Name, Namespace: rp.Namespace}, &found)) + require.NoError(t, c.Delete(ctx, &found, client.PropagationPolicy(metav1.DeletePropagationOrphan))) + + // The orphan-delete leaves no STS behind; if a racing pre-pause + // reconcile recreated it, fail fast with a clear message. + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var sts appsv1.StatefulSet + err := c.Get(ctx, client.ObjectKey{Name: rp.Name, Namespace: rp.Namespace}, &sts) + assert.True(ct, apierrors.IsNotFound(err), "StatefulSet still exists (recreated by a racing reconcile?)") + }, time.Minute, time.Second) + + var brokers []*redpandav1alpha2.Broker + for i := range replicas { + podName := fmt.Sprintf("%s-%d", rp.Name, i) + var pod corev1.Pod + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: podName, Namespace: rp.Namespace}, &pod)) + + podSpec := pod.Spec + podSpec.NodeName = "" + + specBytes, err := yaml.Marshal(podSpec) + require.NoError(t, err) + checksum := fmt.Sprintf("%x", sha256.Sum256(specBytes)) + + storage := redpandav1alpha2.BrokerStorage{} + if opts.useVolumeClaimTemplates { + for _, vol := range pod.Spec.Volumes { + if vol.PersistentVolumeClaim == nil { + continue + } + pvcName := vol.PersistentVolumeClaim.ClaimName + vctName := strings.TrimSuffix(pvcName, "-"+podName) + + var pvc corev1.PersistentVolumeClaim + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: pvcName, Namespace: rp.Namespace}, &pvc)) + + pvcSpec := pvc.Spec + pvcSpec.VolumeName = "" + storage.VolumeClaimTemplates = append(storage.VolumeClaimTemplates, redpandav1alpha2.BrokerVolumeClaim{ + Name: vctName, + Spec: pvcSpec, + }) + } + } else { + for _, vol := range pod.Spec.Volumes { + if vol.PersistentVolumeClaim != nil { + storage.ExistingClaims = append(storage.ExistingClaims, redpandav1alpha2.ExistingClaim{ + Name: vol.PersistentVolumeClaim.ClaimName, + }) + } + } + } + + broker := &redpandav1alpha2.Broker{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", rp.Name, i), + Labels: map[string]string{ + redpandav1alpha2.ClusterNameLabel: rp.Name, + }, + }, + Spec: redpandav1alpha2.BrokerSpec{ + ClusterRef: redpandav1alpha2.ClusterRef{Name: rp.Name}, + NetworkIndex: ptr.To(int32(i)), + PodTemplate: redpandav1alpha2.BrokerPodTemplate{ + Labels: pod.Labels, + Annotations: map[string]string{"config.redpanda.com/checksum": checksum}, + Spec: podSpec, + }, + Storage: storage, + }, + } + require.NoError(t, c.Create(ctx, broker)) + brokers = append(brokers, broker) + } + + deadline := strconv.FormatInt(time.Now().Add(opts.grantDuration).Unix(), 10) + for _, b := range brokers { + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(b), b)) + p := client.MergeFrom(b.DeepCopy()) + if b.Annotations == nil { + b.Annotations = map[string]string{} + } + checksum := b.Spec.PodTemplate.Annotations["config.redpanda.com/checksum"] + b.Annotations["operator.redpanda.com/roll-grant"] = checksum + "/" + deadline + require.NoError(t, c.Patch(ctx, b, p)) + } + + for _, b := range brokers { + s.waitForPhase(t, ctx, c, b, redpandav1alpha2.BrokerPhaseRunning) + } + + return rp, brokers +} + +func (s *BrokerControllerSuite) minimalBroker(clusterName string) *redpandav1alpha2.Broker { + return &redpandav1alpha2.Broker{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName + "-0", + Labels: map[string]string{ + redpandav1alpha2.ClusterNameLabel: clusterName, + }, + }, + Spec: redpandav1alpha2.BrokerSpec{ + ClusterRef: redpandav1alpha2.ClusterRef{Name: clusterName}, + NetworkIndex: ptr.To(int32(0)), + PodTemplate: redpandav1alpha2.BrokerPodTemplate{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "redpanda", + Image: "busybox", + Command: []string{"sleep", "3600"}, + }}, + }, + }, + }, + } +} + +func (s *BrokerControllerSuite) minimalRP() *redpandav1alpha2.Redpanda { + rp := &redpandav1alpha2.Redpanda{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rp-" + testenv.RandString(6), + Annotations: make(map[string]string), + }, + Spec: redpandav1alpha2.MinimalRedpandaSpec(), + } + rp.Spec.ClusterSpec.Image.Repository = ptr.To(os.Getenv("TEST_REDPANDA_REPO")) + return rp +} + +func (s *BrokerControllerSuite) waitForFinalizer(t testing.TB, ctx context.Context, c client.Client, broker *redpandav1alpha2.Broker) { + require.EventuallyWithT(t, func(ct *assert.CollectT) { + assert.NoError(ct, c.Get(ctx, client.ObjectKeyFromObject(broker), broker)) + assert.True(ct, controllerutil.ContainsFinalizer(broker, "cluster.redpanda.com/broker-decommission"), + "broker %q missing finalizer", broker.Name) + }, 2*time.Minute, time.Second) +} + +func (s *BrokerControllerSuite) waitForDeletion(t testing.TB, ctx context.Context, c client.Client, broker *redpandav1alpha2.Broker) { + err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 5*time.Minute, false, func(ctx context.Context) (bool, error) { + err := c.Get(ctx, client.ObjectKeyFromObject(broker), broker) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + t.Logf("Broker %q still exists (phase=%s, finalizers=%v)", broker.Name, broker.Status.Phase, broker.Finalizers) + return false, nil + }) + require.NoError(t, err, "Broker %q was not deleted", broker.Name) +} + +func (s *BrokerControllerSuite) waitForPhase(t testing.TB, ctx context.Context, c client.Client, broker *redpandav1alpha2.Broker, phase redpandav1alpha2.BrokerPhase) { + require.EventuallyWithT(t, func(ct *assert.CollectT) { + assert.NoError(ct, c.Get(ctx, client.ObjectKeyFromObject(broker), broker)) + t.Logf("Broker %q phase=%s (want %s)", broker.Name, broker.Status.Phase, phase) + assert.Equal(ct, phase, broker.Status.Phase) + }, 5*time.Minute, 5*time.Second) +} + +func (s *BrokerControllerSuite) applyAndWait(t testing.TB, ctx context.Context, c client.Client, objs ...client.Object) { + for _, obj := range objs { + gvk, err := c.GroupVersionKindFor(obj) + require.NoError(t, err) + obj.SetManagedFields(nil) + obj.SetResourceVersion("") + obj.GetObjectKind().SetGroupVersionKind(gvk) + require.NoError(t, c.Patch(ctx, obj, client.Apply, client.ForceOwnership, client.FieldOwner("tests"))) //nolint:staticcheck // TODO: migrate to client.Client.Apply() + } + for _, obj := range objs { + switch obj := obj.(type) { + case *redpandav1alpha2.Redpanda: + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 10*time.Minute, false, func(ctx context.Context) (bool, error) { + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { + return false, err + } + for _, cond := range obj.Status.Conditions { + if cond.Type == "Stable" && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == obj.Generation { + return true, nil + } + } + t.Logf("waiting for Redpanda %q to stabilize (gen=%d)", obj.Name, obj.Generation) + return false, nil + }) + require.NoError(t, err) + case *corev1.Secret, *corev1.ConfigMap, *corev1.ServiceAccount, + *rbacv1.ClusterRole, *rbacv1.Role, *rbacv1.RoleBinding, *rbacv1.ClusterRoleBinding: + // no wait needed + default: + t.Fatalf("unhandled object %T in applyAndWait", obj) + } + } +} diff --git a/operator/internal/testenv/testenv.go b/operator/internal/testenv/testenv.go index 11e168c3ff..e31ec5cbb5 100644 --- a/operator/internal/testenv/testenv.go +++ b/operator/internal/testenv/testenv.go @@ -229,6 +229,10 @@ func (e *Env) RESTConfig() *rest.Config { return e.config } +func (e *Env) Host() *k3d.Cluster { + return e.host +} + func (e *Env) Namespace() string { return e.namespace.Name } From 3426d2aec930f1a79662dfe1d9edb5930c0c6935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Thu, 9 Jul 2026 12:09:01 +0200 Subject: [PATCH 04/29] operator: add Broker CR acceptance coverage One end-to-end scenario against a real cluster: pods are adopted from an orphan-deleted StatefulSet without a restart (adoption and pod-ensure are not grant-gated), a targeted roll-grant rotates exactly one broker onto an updated pod template, and an explicit decommission drains the broker out of the cluster. The shared operator runs with --enable-broker; Broker CRs are created by the test itself, standing in for the owning cluster controller that arrives in a follow-up. Co-Authored-By: Claude Fable 5 --- .../operator-Added-20260709-120000.yaml | 4 + .../features/broker-crd-adoption.feature | 20 ++ acceptance/go.mod | 14 + acceptance/main_test.go | 1 + acceptance/steps/broker.go | 253 ++++++++++++++++++ acceptance/steps/register.go | 12 + .../testdata/template-cases.golden.txtar | 44 +-- 7 files changed, 326 insertions(+), 22 deletions(-) create mode 100644 .changes/unreleased/operator-Added-20260709-120000.yaml create mode 100644 acceptance/features/broker-crd-adoption.feature create mode 100644 acceptance/steps/broker.go diff --git a/.changes/unreleased/operator-Added-20260709-120000.yaml b/.changes/unreleased/operator-Added-20260709-120000.yaml new file mode 100644 index 0000000000..cb5765076e --- /dev/null +++ b/.changes/unreleased/operator-Added-20260709-120000.yaml @@ -0,0 +1,4 @@ +project: operator +kind: Added +body: Broker custom resource and reconciler (experimental, behind --enable-broker) — per-broker pod/PVC lifecycle with grant-gated serialized rotation, leadership drain, node_id continuity verification, explicit-intent decommission with recommission and last-broker guard, dead-node PV remediation, and full status conditions +time: 2026-07-09T12:00:00.000000000+02:00 diff --git a/acceptance/features/broker-crd-adoption.feature b/acceptance/features/broker-crd-adoption.feature new file mode 100644 index 0000000000..66dbc39d47 --- /dev/null +++ b/acceptance/features/broker-crd-adoption.feature @@ -0,0 +1,20 @@ +@serial +Feature: Broker CRD pod adoption + @skip:gke @skip:aks @skip:eks + Scenario: Adopt pods from orphan-deleted StatefulSet and decommission a broker + Given I create a basic cluster "broker-test" with 3 nodes + And cluster "broker-test" is stable with 3 nodes + When I pause reconciliation on cluster "broker-test" + And I orphan-delete the StatefulSet for cluster "broker-test" + And I create Broker CRs for cluster "broker-test" + # Adoption and pod-ensure are not gated on roll-grants (RFC Q5). + Then all Broker CRs for cluster "broker-test" should be Running + And cluster "broker-test" admin API should show 3 brokers + When I update Broker "broker-test-0" pod template with env "ROTATION_TEST=applied" for cluster "broker-test" + # Rotation IS gated: only the targeted broker gets a grant. + And I grant a roll-grant to Broker "broker-test-0" + Then Broker "broker-test-0" pod should have env "ROTATION_TEST" = "applied" + And all Broker CRs for cluster "broker-test" should be Running + When I set decommission on Broker "broker-test-2" + Then Broker "broker-test-2" should reach phase "Decommissioned" + And cluster "broker-test" admin API should show 2 brokers diff --git a/acceptance/go.mod b/acceptance/go.mod index 22c16bfc83..dd719e7332 100644 --- a/acceptance/go.mod +++ b/acceptance/go.mod @@ -29,10 +29,15 @@ require ( require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260209202127-80ab13bee0bf.1 // indirect buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.36.10-20221127060915-a1ecdc58eccd.1 // indirect + buf.build/gen/go/redpandadata/cloud/connectrpc/go v1.18.1-20250616170632-3de895655308.1 // indirect buf.build/gen/go/redpandadata/cloud/protocolbuffers/go v1.36.10-20251124181447-cf877111f58c.1 // indirect buf.build/gen/go/redpandadata/common/protocolbuffers/go v1.36.11-20260316210807-5d899910f714.1 // indirect buf.build/gen/go/redpandadata/core/connectrpc/go v1.19.2-20260506141738-8dfb14358799.1 // indirect buf.build/gen/go/redpandadata/core/protocolbuffers/go v1.36.11-20260506141738-8dfb14358799.1 // indirect + buf.build/gen/go/redpandadata/dataplane/connectrpc/go v1.18.1-20250404200318-65f29ddd7b29.1 // indirect + buf.build/gen/go/redpandadata/dataplane/protocolbuffers/go v1.36.5-20250404200318-65f29ddd7b29.1 // indirect + buf.build/gen/go/redpandadata/gatekeeper/connectrpc/go v1.18.1-20241209180130-05cf059c71c1.1 // indirect + buf.build/gen/go/redpandadata/gatekeeper/protocolbuffers/go v1.36.6-20241209180130-05cf059c71c1.1 // indirect cel.dev/expr v0.25.1 // indirect cloud.google.com/go/auth v0.18.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -96,6 +101,7 @@ require ( github.com/cucumber/messages/go/v21 v21.0.1 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect @@ -128,6 +134,7 @@ require ( github.com/go-openapi/swag/yamlutils v0.25.4 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/gobwas/glob v0.2.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect @@ -185,6 +192,12 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/lestrrat-go/blackmagic v1.0.2 // indirect + github.com/lestrrat-go/httpcc v1.0.1 // indirect + github.com/lestrrat-go/httprc v1.0.6 // indirect + github.com/lestrrat-go/iter v1.0.2 // indirect + github.com/lestrrat-go/jwx/v2 v2.1.5-0.20250226052408-de7d95fda31a // indirect + github.com/lestrrat-go/option v1.0.1 // indirect github.com/lib/pq v1.10.9 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect @@ -237,6 +250,7 @@ require ( github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/scalalang2/golang-fifo v1.2.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect github.com/sergi/go-diff v1.4.0 // indirect github.com/sethgrid/pester v1.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect diff --git a/acceptance/main_test.go b/acceptance/main_test.go index b202a0b3d9..7c48e039ea 100644 --- a/acceptance/main_test.go +++ b/acceptance/main_test.go @@ -384,6 +384,7 @@ func operatorCmdFlags() []string { "--unbind-pvcs-after=5s", "--cluster-connection-timeout=500ms", "--enable-shadowlinks", + "--enable-broker", } if testutil.AcceptanceSetupNodePools() > 0 { flags = append(flags, "--enable-v2-nodepools=true") diff --git a/acceptance/steps/broker.go b/acceptance/steps/broker.go new file mode 100644 index 0000000000..8b076d8930 --- /dev/null +++ b/acceptance/steps/broker.go @@ -0,0 +1,253 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package steps + +import ( + "context" + "crypto/sha256" + "fmt" + "sort" + "strings" + "time" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + framework "github.com/redpanda-data/redpanda-operator/harpoon" + redpandav1alpha2 "github.com/redpanda-data/redpanda-operator/operator/api/redpanda/v1alpha2" + "github.com/redpanda-data/redpanda-operator/operator/pkg/feature" +) + +func pauseReconciliation(ctx context.Context, t framework.TestingT, clusterName string) { + key := t.ResourceKey(clusterName) + var cluster redpandav1alpha2.Redpanda + require.NoError(t, t.Get(ctx, key, &cluster)) + + patch := runtimeclient.MergeFrom(cluster.DeepCopy()) + if cluster.Annotations == nil { + cluster.Annotations = map[string]string{} + } + cluster.Annotations["cluster.redpanda.com/managed"] = "false" + require.NoError(t, t.Patch(ctx, &cluster, patch)) + t.Logf("Paused reconciliation on cluster %q", clusterName) +} + +func orphanDeleteStatefulSet(ctx context.Context, t framework.TestingT, clusterName string) { + key := t.ResourceKey(clusterName) + var sts appsv1.StatefulSet + require.NoError(t, t.Get(ctx, key, &sts)) + require.NoError(t, t.Delete(ctx, &sts, runtimeclient.PropagationPolicy(metav1.DeletePropagationOrphan))) + t.Logf("Orphan-deleted StatefulSet %q", clusterName) +} + +func createBrokerCRsForCluster(ctx context.Context, t framework.TestingT, clusterName string) { + key := t.ResourceKey(clusterName) + + var sts appsv1.StatefulSet + // STS may already be gone if orphan-delete propagated; list pods by label instead. + var pods corev1.PodList + err := t.Get(ctx, key, &sts) + if err == nil { + sel, selErr := metav1.LabelSelectorAsSelector(sts.Spec.Selector) + require.NoError(t, selErr) + require.NoError(t, t.List(ctx, &pods, runtimeclient.InNamespace(key.Namespace), runtimeclient.MatchingLabelsSelector{Selector: sel})) + } else { + require.NoError(t, t.List(ctx, &pods, runtimeclient.InNamespace(key.Namespace), runtimeclient.MatchingLabels{ + "app.kubernetes.io/instance": clusterName, + "app.kubernetes.io/name": "redpanda", + })) + } + require.NotEmpty(t, pods.Items, "no pods found for cluster %q", clusterName) + + sort.Slice(pods.Items, func(i, j int) bool { return pods.Items[i].Name < pods.Items[j].Name }) + + for i, pod := range pods.Items { + brokerName := fmt.Sprintf("%s-%d", key.Name, i) + + specBytes, err := yaml.Marshal(pod.Spec) + require.NoError(t, err) + checksum := fmt.Sprintf("%x", sha256.Sum256(specBytes)) + + // Collect existing PVC names from the pod's volumes. + // For StatefulSet pods, the ClaimName is already the full PVC name + // (e.g. "datadir-broker-test-0"), not the VolumeClaimTemplate name. + var existingClaims []redpandav1alpha2.ExistingClaim + for _, vol := range pod.Spec.Volumes { + if vol.PersistentVolumeClaim != nil { + existingClaims = append(existingClaims, redpandav1alpha2.ExistingClaim{ + Name: vol.PersistentVolumeClaim.ClaimName, + }) + } + } + + broker := &redpandav1alpha2.Broker{ + ObjectMeta: metav1.ObjectMeta{ + Name: brokerName, + Namespace: key.Namespace, + Labels: map[string]string{ + redpandav1alpha2.ClusterNameLabel: clusterName, + }, + }, + Spec: redpandav1alpha2.BrokerSpec{ + ClusterRef: redpandav1alpha2.ClusterRef{ + Name: clusterName, + }, + NetworkIndex: ptr.To(int32(i)), + PodTemplate: redpandav1alpha2.BrokerPodTemplate{ + Labels: pod.Labels, + Annotations: map[string]string{redpandav1alpha2.BrokerConfigChecksumAnnotation: checksum}, + Spec: pod.Spec, + }, + Storage: redpandav1alpha2.BrokerStorage{ + ExistingClaims: existingClaims, + }, + }, + } + require.NoError(t, t.Create(ctx, broker)) + t.Logf("Created Broker CR %q (networkIndex=%d)", brokerName, i) + } + + t.Cleanup(func(ctx context.Context) { + t := framework.T(ctx) + var brokers redpandav1alpha2.BrokerList + _ = t.List(ctx, &brokers, runtimeclient.InNamespace(key.Namespace)) + for i := range brokers.Items { + _ = t.Delete(ctx, &brokers.Items[i]) + } + }) +} + +func grantRollGrantToBroker(ctx context.Context, t framework.TestingT, brokerName string) { + key := t.ResourceKey(brokerName) + var broker redpandav1alpha2.Broker + require.NoError(t, t.Get(ctx, key, &broker)) + + checksum := broker.Spec.PodTemplate.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation] + require.NotEmpty(t, checksum, "broker %q missing config checksum", brokerName) + + patch := runtimeclient.MergeFrom(broker.DeepCopy()) + if broker.Annotations == nil { + broker.Annotations = map[string]string{} + } + broker.Annotations[feature.RollGrant.Key] = feature.FormatRollGrant(checksum, time.Now().Add(feature.RollGrantTTL)) + require.NoError(t, t.Patch(ctx, &broker, patch)) + t.Logf("Granted roll-grant to Broker %q", brokerName) +} + +func allBrokerCRsRunning(ctx context.Context, t framework.TestingT, clusterName string) { + key := t.ResourceKey(clusterName) + require.Eventually(t, func() bool { + var brokers redpandav1alpha2.BrokerList + if err := t.List(ctx, &brokers, runtimeclient.InNamespace(key.Namespace)); err != nil { + return false + } + if len(brokers.Items) == 0 { + return false + } + for _, b := range brokers.Items { + if b.Status.Phase != redpandav1alpha2.BrokerPhaseRunning { + t.Logf("Broker %q phase=%s (want Running)", b.Name, b.Status.Phase) + return false + } + } + return true + }, 5*time.Minute, 5*time.Second, "not all Broker CRs reached Running phase") +} + +func setDecommissionOnBroker(ctx context.Context, t framework.TestingT, brokerName string) { + key := t.ResourceKey(brokerName) + var broker redpandav1alpha2.Broker + require.NoError(t, t.Get(ctx, key, &broker)) + + patch := runtimeclient.MergeFrom(broker.DeepCopy()) + broker.Spec.Decommission = true + require.NoError(t, t.Patch(ctx, &broker, patch)) + t.Logf("Set decommission=true on Broker %q", brokerName) +} + +func brokerShouldReachPhase(ctx context.Context, t framework.TestingT, brokerName, phase string) { + key := t.ResourceKey(brokerName) + var broker redpandav1alpha2.Broker + require.Eventually(t, func() bool { + if err := t.Get(ctx, key, &broker); err != nil { + return false + } + t.Logf("Broker %q phase=%s (want %s)", brokerName, broker.Status.Phase, phase) + return string(broker.Status.Phase) == phase + }, 5*time.Minute, 5*time.Second, "Broker %q never reached phase %q", brokerName, phase) +} + +func updateBrokerPodTemplateEnv(ctx context.Context, t framework.TestingT, brokerName, envKeyValue, _ string) { + key := t.ResourceKey(brokerName) + var broker redpandav1alpha2.Broker + require.NoError(t, t.Get(ctx, key, &broker)) + + parts := strings.SplitN(envKeyValue, "=", 2) + require.Len(t, parts, 2, "env must be KEY=VALUE, got %q", envKeyValue) + + broker.Spec.PodTemplate.Spec.Containers[0].Env = append( + broker.Spec.PodTemplate.Spec.Containers[0].Env, + corev1.EnvVar{Name: parts[0], Value: parts[1]}, + ) + + specBytes, err := yaml.Marshal(broker.Spec.PodTemplate.Spec) + require.NoError(t, err) + checksum := fmt.Sprintf("%x", sha256.Sum256(specBytes)) + broker.Spec.PodTemplate.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation] = checksum + + require.NoError(t, t.Update(ctx, &broker)) + t.Logf("Updated Broker %q pod template: added env %s, new checksum %s", brokerName, envKeyValue, checksum[:12]) +} + +func brokerPodShouldHaveEnv(ctx context.Context, t framework.TestingT, brokerName, envName, envValue string) { + key := t.ResourceKey(brokerName) + require.Eventually(t, func() bool { + var broker redpandav1alpha2.Broker + if err := t.Get(ctx, key, &broker); err != nil { + return false + } + var pod corev1.Pod + if err := t.Get(ctx, runtimeclient.ObjectKey{Name: broker.PodName(), Namespace: key.Namespace}, &pod); err != nil { + return false + } + for _, c := range pod.Spec.Containers { + for _, e := range c.Env { + if e.Name == envName && e.Value == envValue { + return true + } + } + } + t.Logf("Broker %q pod %q does not yet have env %s=%s", brokerName, pod.Name, envName, envValue) + return false + }, 5*time.Minute, 5*time.Second, "Broker %q pod never got env %s=%s", brokerName, envName, envValue) +} + +func clusterAdminAPIShouldShowBrokers(ctx context.Context, t framework.TestingT, clusterName string, count int) { + clients := clientsForCluster(ctx, clusterName) + require.Eventually(t, func() bool { + admin, err := clients.factory.RedpandaAdminClient(ctx, clients.resourceTarget) + if err != nil { + t.Logf("Failed to create admin client: %v", err) + return false + } + brokers, err := admin.Brokers(ctx) + if err != nil { + t.Logf("Failed to get brokers: %v", err) + return false + } + t.Logf("Cluster %q has %d brokers (want %d)", clusterName, len(brokers), count) + return len(brokers) == count + }, 5*time.Minute, 5*time.Second, "cluster %q never had %d brokers", clusterName, count) +} diff --git a/acceptance/steps/register.go b/acceptance/steps/register.go index 62f0317084..e5ca8821c3 100644 --- a/acceptance/steps/register.go +++ b/acceptance/steps/register.go @@ -176,6 +176,18 @@ func init() { framework.RegisterStep(`^service "([^"]*)" should not have field managers:$`, checkResourceNoFieldManagers) framework.RegisterStep(`^cluster "([^"]*)" should have sync error:$`, checkClusterHasSyncError) + // Broker CRD scenario steps + framework.RegisterStep(`^I pause reconciliation on cluster "([^"]*)"$`, pauseReconciliation) + framework.RegisterStep(`^I orphan-delete the StatefulSet for cluster "([^"]*)"$`, orphanDeleteStatefulSet) + framework.RegisterStep(`^I create Broker CRs for cluster "([^"]*)"$`, createBrokerCRsForCluster) + framework.RegisterStep(`^I grant a roll-grant to Broker "([^"]*)"$`, grantRollGrantToBroker) + framework.RegisterStep(`^all Broker CRs for cluster "([^"]*)" should be Running$`, allBrokerCRsRunning) + framework.RegisterStep(`^I set decommission on Broker "([^"]*)"$`, setDecommissionOnBroker) + framework.RegisterStep(`^Broker "([^"]*)" should reach phase "([^"]*)"$`, brokerShouldReachPhase) + framework.RegisterStep(`^cluster "([^"]*)" admin API should show (\d+) brokers$`, clusterAdminAPIShouldShowBrokers) + framework.RegisterStep(`^I update Broker "([^"]*)" pod template with env "([^"]*)" for cluster "([^"]*)"$`, updateBrokerPodTemplateEnv) + framework.RegisterStep(`^Broker "([^"]*)" pod should have env "([^"]*)" = "([^"]*)"$`, brokerPodShouldHaveEnv) + // Debug steps framework.RegisterStep(`^I become debuggable$`, sleepALongTime) } diff --git a/charts/redpanda/testdata/template-cases.golden.txtar b/charts/redpanda/testdata/template-cases.golden.txtar index beb6412912..284195f5fa 100644 --- a/charts/redpanda/testdata/template-cases.golden.txtar +++ b/charts/redpanda/testdata/template-cases.golden.txtar @@ -141574,8 +141574,8 @@ metadata: app.kubernetes.io/instance: redpanda app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: console - app.kubernetes.io/version: v3.7.0 - helm.sh/chart: console-3.7.0 + app.kubernetes.io/version: v3.8.0 + helm.sh/chart: console-3.8.0 name: redpanda-console namespace: default --- @@ -142000,8 +142000,8 @@ metadata: app.kubernetes.io/instance: redpanda app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: console - app.kubernetes.io/version: v3.7.0 - helm.sh/chart: console-3.7.0 + app.kubernetes.io/version: v3.8.0 + helm.sh/chart: console-3.8.0 name: redpanda-console namespace: default --- @@ -142117,8 +142117,8 @@ metadata: app.kubernetes.io/instance: redpanda app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: console - app.kubernetes.io/version: v3.7.0 - helm.sh/chart: console-3.7.0 + app.kubernetes.io/version: v3.8.0 + helm.sh/chart: console-3.8.0 name: redpanda-console namespace: default spec: @@ -142228,8 +142228,8 @@ metadata: app.kubernetes.io/instance: redpanda app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: console - app.kubernetes.io/version: v3.7.0 - helm.sh/chart: console-3.7.0 + app.kubernetes.io/version: v3.8.0 + helm.sh/chart: console-3.8.0 name: redpanda-console namespace: default spec: @@ -142257,15 +142257,15 @@ spec: - name: REDPANDA_METRICS_K8S_DEPLOYMENT_TYPE value: helm - name: REDPANDA_METRICS_K8S_CHART_VERSION - value: 3.7.0 + value: 3.8.0 - name: REDPANDA_METRICS_K8S_CONSOLE_IMAGE_VERSION - value: redpandadata/console:v3.7.0 + value: redpandadata/console:v3.8.0 - name: REDPANDA_METRICS_K8S_VERSION value: v1.99.0-gke - name: REDPANDA_METRICS_K8S_ENVIRONMENT value: GCP envFrom: [] - image: docker.redpanda.com/redpandadata/console:v3.7.0 + image: docker.redpanda.com/redpandadata/console:v3.8.0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 @@ -143152,8 +143152,8 @@ metadata: app.kubernetes.io/instance: redpanda app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: console - app.kubernetes.io/version: v3.7.0 - helm.sh/chart: console-3.7.0 + app.kubernetes.io/version: v3.8.0 + helm.sh/chart: console-3.8.0 name: redpanda-console namespace: default --- @@ -143578,8 +143578,8 @@ metadata: app.kubernetes.io/instance: redpanda app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: console - app.kubernetes.io/version: v3.7.0 - helm.sh/chart: console-3.7.0 + app.kubernetes.io/version: v3.8.0 + helm.sh/chart: console-3.8.0 name: redpanda-console namespace: default --- @@ -143695,8 +143695,8 @@ metadata: app.kubernetes.io/instance: redpanda app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: console - app.kubernetes.io/version: v3.7.0 - helm.sh/chart: console-3.7.0 + app.kubernetes.io/version: v3.8.0 + helm.sh/chart: console-3.8.0 name: redpanda-console namespace: default spec: @@ -143806,8 +143806,8 @@ metadata: app.kubernetes.io/instance: redpanda app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: console - app.kubernetes.io/version: v3.7.0 - helm.sh/chart: console-3.7.0 + app.kubernetes.io/version: v3.8.0 + helm.sh/chart: console-3.8.0 name: redpanda-console namespace: default spec: @@ -143835,15 +143835,15 @@ spec: - name: REDPANDA_METRICS_K8S_DEPLOYMENT_TYPE value: helm - name: REDPANDA_METRICS_K8S_CHART_VERSION - value: 3.7.0 + value: 3.8.0 - name: REDPANDA_METRICS_K8S_CONSOLE_IMAGE_VERSION - value: redpandadata/console:v3.7.0 + value: redpandadata/console:v3.8.0 - name: REDPANDA_METRICS_K8S_VERSION value: v1.99.0-gke - name: REDPANDA_METRICS_K8S_ENVIRONMENT value: GCP envFrom: [] - image: docker.redpanda.com/redpandadata/console:v3.7.0 + image: docker.redpanda.com/redpandadata/console:v3.8.0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 From 39b63af156ef3a8ffcb93ad862b218b673ef2fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 14:44:20 +0200 Subject: [PATCH 05/29] fixup! operator: add the Broker CR reconciler --- .../controller/redpanda/broker_controller.go | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index 97d5874d01..96f39833de 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -93,6 +93,11 @@ type brokerReconciliationState struct { pod *corev1.Pod // nil when pod does not exist yet phase redpandav1alpha2.BrokerPhase // empty = compute from pod status granted bool + // clusterName is the multicluster-runtime cluster this request came + // from. Admin-API clients must be built for it: the Broker's pods live + // there, and targeting the local cluster's Redpanda instead would + // register/decommission/drain against the wrong cluster. + clusterName string // initialStatus snapshots Status as fetched, so syncBrokerStatus can // skip the API write when nothing changed (RFC Q11: rate-limit status // updates). @@ -154,13 +159,14 @@ func (r *BrokerReconciler) Reconcile(ctx context.Context, req mcreconcile.Reques } } } else { - return r.reconcileDelete(ctx, l, k8sClient, &broker, broker.PodName()) + return r.reconcileDelete(ctx, l, k8sClient, req.ClusterName, &broker, broker.PodName()) } state, err := r.fetchState(ctx, k8sClient, &broker) if err != nil { return ctrl.Result{}, err } + state.clusterName = req.ClusterName reconcilers := []brokerReconcilerFn{ r.reconcilePVCs, @@ -364,7 +370,7 @@ func (r *BrokerReconciler) reconcilePodRotation(ctx context.Context, state *brok return ctrl.Result{RequeueAfter: periodicRequeue}, nil } if broker.Status.BrokerID != nil { - drained, err := r.ensureDrained(ctx, broker) + drained, err := r.ensureDrained(ctx, state.clusterName, broker) if err != nil { return ctrl.Result{}, fmt.Errorf("draining broker %d: %w", *broker.Status.BrokerID, err) } @@ -409,7 +415,7 @@ func (r *BrokerReconciler) reconcileBrokerRegistration(ctx context.Context, stat l := log.FromContext(ctx) podName := broker.PodName() - currentID, err := r.resolveBrokerID(ctx, broker, podName) + currentID, err := r.resolveBrokerID(ctx, state.clusterName, broker, podName) if err != nil { l.Info("could not resolve broker ID, will retry", "error", err) return ctrl.Result{RequeueAfter: requeueShort}, nil @@ -442,7 +448,7 @@ func (r *BrokerReconciler) reconcileBrokerRegistration(ctx context.Context, stat } state.registrationVerified = true - if err := r.disableMaintenanceMode(ctx, broker); err != nil { + if err := r.disableMaintenanceMode(ctx, state.clusterName, broker); err != nil { l.Info("could not disable maintenance mode", "error", err) } @@ -471,7 +477,7 @@ func (r *BrokerReconciler) reconcileDecommission(ctx context.Context, state *bro return ctrl.Result{}, nil } - decommResult, err := r.executeDecommission(ctx, broker) + decommResult, err := r.executeDecommission(ctx, state.clusterName, broker) if err != nil { return ctrl.Result{}, err } @@ -669,7 +675,7 @@ func (r *BrokerReconciler) executeRecommission(ctx context.Context, state *broke broker := state.broker brokerID := int(*broker.Status.BrokerID) - admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) + admin, err := r.ClientFactory.RedpandaAdminClientForCluster(ctx, broker, state.clusterName) if err != nil { return ctrl.Result{}, err } @@ -694,11 +700,11 @@ type decommissionResult struct { requeue bool } -func (r *BrokerReconciler) executeDecommission(ctx context.Context, broker *redpandav1alpha2.Broker) (decommissionResult, error) { +func (r *BrokerReconciler) executeDecommission(ctx context.Context, clusterName string, broker *redpandav1alpha2.Broker) (decommissionResult, error) { l := log.FromContext(ctx) brokerID := int(*broker.Status.BrokerID) - admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) + admin, err := r.ClientFactory.RedpandaAdminClientForCluster(ctx, broker, clusterName) if err != nil { return decommissionResult{phase: redpandav1alpha2.BrokerPhaseDecommissioning}, err } @@ -748,7 +754,7 @@ func (r *BrokerReconciler) executeDecommission(ctx context.Context, broker *redp // deletion policy decides: "cascade" (default) lets the GC delete pod and // PVCs with the CR — whole-cluster teardown, decommission is pointless — // while "orphan" releases them. -func (r *BrokerReconciler) reconcileDelete(ctx context.Context, l logr.Logger, k8sClient client.Client, broker *redpandav1alpha2.Broker, podName string) (ctrl.Result, error) { +func (r *BrokerReconciler) reconcileDelete(ctx context.Context, l logr.Logger, k8sClient client.Client, clusterName string, broker *redpandav1alpha2.Broker, podName string) (ctrl.Result, error) { if !controllerutil.ContainsFinalizer(broker, brokerFinalizerName) { return ctrl.Result{}, nil } @@ -773,7 +779,7 @@ func (r *BrokerReconciler) reconcileDelete(ctx context.Context, l logr.Logger, k // it live rather than skipping the decommission and leaving a dead // membership entry behind. if broker.Status.BrokerID == nil { - brokerID, err := r.resolveBrokerID(ctx, broker, podName) + brokerID, err := r.resolveBrokerID(ctx, clusterName, broker, podName) if err != nil { l.Info("could not resolve broker ID before decommission, will retry", "error", err) return ctrl.Result{RequeueAfter: requeueShort}, nil @@ -781,7 +787,7 @@ func (r *BrokerReconciler) reconcileDelete(ctx context.Context, l logr.Logger, k broker.Status.BrokerID = brokerID } if broker.Status.BrokerID != nil { - result, err := r.executeDecommission(ctx, broker) + result, err := r.executeDecommission(ctx, clusterName, broker) if err != nil { return ctrl.Result{}, err } @@ -921,11 +927,11 @@ func removeOwnerRefByUID(obj client.Object, uid types.UID) bool { // ensureDrained enables maintenance mode and returns true when leadership drain // is complete. Callers should requeue until this returns true. -func (r *BrokerReconciler) ensureDrained(ctx context.Context, broker *redpandav1alpha2.Broker) (bool, error) { +func (r *BrokerReconciler) ensureDrained(ctx context.Context, clusterName string, broker *redpandav1alpha2.Broker) (bool, error) { l := log.FromContext(ctx) brokerID := int(*broker.Status.BrokerID) - admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) + admin, err := r.ClientFactory.RedpandaAdminClientForCluster(ctx, broker, clusterName) if err != nil { return false, err } @@ -1028,8 +1034,8 @@ func (r *BrokerReconciler) remediatePVAffinity(ctx context.Context, l logr.Logge return true, nil } -func (r *BrokerReconciler) disableMaintenanceMode(ctx context.Context, broker *redpandav1alpha2.Broker) error { - admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) +func (r *BrokerReconciler) disableMaintenanceMode(ctx context.Context, clusterName string, broker *redpandav1alpha2.Broker) error { + admin, err := r.ClientFactory.RedpandaAdminClientForCluster(ctx, broker, clusterName) if err != nil { return err } @@ -1037,8 +1043,8 @@ func (r *BrokerReconciler) disableMaintenanceMode(ctx context.Context, broker *r return admin.DisableMaintenanceMode(ctx, int(*broker.Status.BrokerID), false) } -func (r *BrokerReconciler) resolveBrokerID(ctx context.Context, broker *redpandav1alpha2.Broker, podName string) (*int32, error) { - admin, err := r.ClientFactory.RedpandaAdminClient(ctx, broker) +func (r *BrokerReconciler) resolveBrokerID(ctx context.Context, clusterName string, broker *redpandav1alpha2.Broker, podName string) (*int32, error) { + admin, err := r.ClientFactory.RedpandaAdminClientForCluster(ctx, broker, clusterName) if err != nil { return nil, err } From af5b5f2ac4cae20c16f58c6192de0ca3c1f64a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 14:50:43 +0200 Subject: [PATCH 06/29] fixup! operator: add the Broker CR reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1: a decommission requested while Status.BrokerID is nil never started — reconcileBrokerRegistration skips decommissioning brokers, so the ID was never resolved and reconcileDecommission only relabeled the phase. Resolve the ID live in the spec path, mirroring the deletion path. --- .../controller/redpanda/broker_controller.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index 96f39833de..d9fd8cc747 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -473,8 +473,20 @@ func (r *BrokerReconciler) reconcileDecommission(ctx context.Context, state *bro state.phase = redpandav1alpha2.BrokerPhaseDecommissioned return ctrl.Result{}, nil } - state.phase = redpandav1alpha2.BrokerPhaseDecommissioning - return ctrl.Result{}, nil + // reconcileBrokerRegistration skips decommissioning brokers, so the + // ID must be resolved live here — mirroring the deletion path — + // or an intent set before the first registration (or right after a + // status-update race) never actually starts the decommission and the + // broker stays a full cluster member while reporting Decommissioning. + currentID, err := r.resolveBrokerID(ctx, state.clusterName, broker, broker.PodName()) + if err != nil || currentID == nil { + if err != nil { + log.FromContext(ctx).Info("could not resolve broker ID before decommission, will retry", "error", err) + } + state.phase = redpandav1alpha2.BrokerPhaseDecommissioning + return ctrl.Result{RequeueAfter: requeueShort}, nil + } + broker.Status.BrokerID = currentID } decommResult, err := r.executeDecommission(ctx, state.clusterName, broker) From 8b0733a2f24393980f921e53bcbde550558a2128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 14:50:43 +0200 Subject: [PATCH 07/29] fixup! operator: add the Broker CR reconciler M2: reconcilePVAffinity and reconcilePodRotation gated only on outdated+grant, so a decommissioning broker could get its pod rotated or its storage remediated mid-drain, interrupting partition movement. Early return on Spec.Decommission like the sibling reconcilers. --- .../controller/redpanda/broker_controller.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index d9fd8cc747..354ebeccd5 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -323,6 +323,13 @@ func (r *BrokerReconciler) reconcilePVCAdoption(ctx context.Context, state *brok } func (r *BrokerReconciler) reconcilePVAffinity(ctx context.Context, state *brokerReconciliationState, cluster cluster.Cluster) (ctrl.Result, error) { + // A decommissioning broker is draining: deleting its pod or discarding + // its storage mid-drain would interrupt partition movement (and clear + // the identity the decommission needs). reconcileDecommission owns the + // pod's fate from here on. + if state.broker.Spec.Decommission { + return ctrl.Result{}, nil + } if state.pod == nil || !pvcunbinder.PodHasVolumeAffinityUnschedulable(state.pod) { return ctrl.Result{}, nil } @@ -355,6 +362,11 @@ func (r *BrokerReconciler) reconcilePodRotation(ctx context.Context, state *brok return ctrl.Result{}, nil } broker := state.broker + // Never rotate a decommissioning broker: the pod must keep running + // (draining) until the decommission completes and deletes it. + if broker.Spec.Decommission { + return ctrl.Result{}, nil + } // PodOutdated covers both the config checksum and the restart-requiring // cluster-config version (see resources.MarkBrokersForRestart); the // recreated pod inherits both from the desired template, clearing the From b1e0bee20e9085773cc9f7957938b841f54ecc74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 14:56:43 +0200 Subject: [PATCH 08/29] fixup! operator: add Broker CRD API, conditions, and roll-grant feature flags M3: BrokerDeletionPolicy.Parse was the identity function while the consumer compares against "orphan" exactly, so any typo ("orphaned", "Orphan", "retain") silently selected the destructive cascade branch. Parse now validates against {cascade, orphan} case-insensitively; unknown values error and Get falls back to the documented default with a logged complaint. --- operator/pkg/feature/flags.go | 12 +++++++++++- operator/pkg/feature/flags_test.go | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/operator/pkg/feature/flags.go b/operator/pkg/feature/flags.go index a2b6a17e83..22639fd127 100644 --- a/operator/pkg/feature/flags.go +++ b/operator/pkg/feature/flags.go @@ -100,8 +100,18 @@ var ( BrokerDeletionPolicy = &AnnotationFeatureFlag[string]{ Key: "operator.redpanda.com/broker-deletion-policy", Default: "cascade", + // Only the two known policies are accepted (case-insensitively). + // Anything else errors so Get falls back to the documented default + // with a logged complaint — a typo like "orphaned" must not be + // silently interpreted as the destructive cascade branch without + // leaving a trace. Parse: func(s string) (string, error) { - return s, nil + switch normalized := strings.ToLower(strings.TrimSpace(s)); normalized { + case "cascade", "orphan": + return normalized, nil + default: + return "", fmt.Errorf("invalid broker deletion policy %q: must be %q or %q", s, "cascade", "orphan") + } }, } ) diff --git a/operator/pkg/feature/flags_test.go b/operator/pkg/feature/flags_test.go index 363e413b14..c158e65f4c 100644 --- a/operator/pkg/feature/flags_test.go +++ b/operator/pkg/feature/flags_test.go @@ -54,3 +54,21 @@ func TestParseRollGrantChecksumWithSlash(t *testing.T) { _, _, ok := ParseRollGrant("abc/def/1234") assert.False(t, ok) } + +func TestBrokerDeletionPolicyParse(t *testing.T) { + t.Parallel() + + for _, valid := range []string{"cascade", "orphan", "Orphan", " CASCADE "} { + parsed, err := BrokerDeletionPolicy.Parse(valid) + require.NoError(t, err, "value %q must parse", valid) + assert.Contains(t, []string{"cascade", "orphan"}, parsed) + } + + // A typo must NOT silently fall through to the destructive cascade + // branch: Parse errors and Get degrades to the default with a logged + // complaint. + for _, invalid := range []string{"orphaned", "retain", "delete", ""} { + _, err := BrokerDeletionPolicy.Parse(invalid) + assert.Error(t, err, "value %q must be rejected", invalid) + } +} From 84632da396a2b253d8863d2091833f126a504392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 14:58:32 +0200 Subject: [PATCH 09/29] fixup! operator: add the Broker CR reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M4: raw CR deletion with the pod already gone removed the finalizer without releasing PVC ownerRefs, so the deletion cascaded and the GC destroyed the data — violating the raw-delete self-heal contract. The pod-absent branch now applies the same deletion-policy decision as the pod-present path; releaseBrokerResources accepts a nil pod. --- .../controller/redpanda/broker_controller.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index 354ebeccd5..e1021c75ac 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -787,7 +787,19 @@ func (r *BrokerReconciler) reconcileDelete(ctx context.Context, l logr.Logger, k err := k8sClient.Get(ctx, client.ObjectKey{Name: podName, Namespace: broker.Namespace}, &pod) switch { case apierrors.IsNotFound(err): - l.Info("pod already gone, removing finalizer") + // The pod is already gone (e.g. deleted out of band), but the PVCs + // may still exist and carry this CR's controller ownerRef. Apply the + // same policy decision as the pod-present path — otherwise the CR + // deletion cascades and the GC deletes the data regardless of + // intent. + if r.deletionPolicy(ctx, l, k8sClient, broker) == deletionPolicyCascade { + l.Info("pod already gone, cascade policy, removing finalizer") + } else { + if err := r.releaseBrokerResources(ctx, l, k8sClient, broker, nil); err != nil { + return ctrl.Result{}, err + } + l.Info("pod already gone, PVCs released, removing finalizer") + } case err != nil: return ctrl.Result{}, err @@ -904,9 +916,10 @@ func (r *BrokerReconciler) deletionPolicy(ctx context.Context, l logr.Logger, k8 } // releaseBrokerResources strips this Broker's ownerRefs from its pod and PVCs -// so they survive the CR deletion. +// so they survive the CR deletion. pod may be nil when it is already gone — +// the PVCs are still released. func (r *BrokerReconciler) releaseBrokerResources(ctx context.Context, l logr.Logger, k8sClient client.Client, broker *redpandav1alpha2.Broker, pod *corev1.Pod) error { - if removeOwnerRefByUID(pod, broker.UID) { + if pod != nil && removeOwnerRefByUID(pod, broker.UID) { l.Info("releasing pod from deleted Broker CR", "pod", pod.Name) if err := k8sClient.Update(ctx, pod); err != nil { return fmt.Errorf("releasing pod %s: %w", pod.Name, err) From 07d1170a1b34649af6f63a52ea103fa6b1552b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 14:59:56 +0200 Subject: [PATCH 10/29] fixup! operator: add the Broker CR reconciler M5: orphan-pod adoption unconditionally stamped the DESIRED config checksum onto the live pod, so a stale pod re-adopted after a raw CR deletion was marked current and never rotated while status reported synced. Adoption now stamps the checksum only when the pod carries none (the migration case, where preconditions verified the pod runs the desired config); an existing live checksum is preserved so PodOutdated can queue the rotation. --- .../controller/redpanda/broker_controller.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index e1021c75ac..16deae561a 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -278,8 +278,17 @@ func (r *BrokerReconciler) reconcilePod(ctx context.Context, state *brokerReconc if pod.Annotations == nil { pod.Annotations = map[string]string{} } - if cs := broker.Spec.PodTemplate.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation]; cs != "" { - pod.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation] = cs + // Stamp the desired checksum ONLY when the pod carries none — the + // STS→Broker migration case, where preconditions verified the pod + // already runs the desired config and adoption must not queue a + // pointless rotation. A pod that already carries a checksum + // (self-heal re-adoption after a raw CR deletion) keeps its live + // value: overwriting it with the desired one would mark a stale pod + // current and silently skip its rotation. + if _, ok := pod.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation]; !ok { + if cs := broker.Spec.PodTemplate.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation]; cs != "" { + pod.Annotations[redpandav1alpha2.BrokerConfigChecksumAnnotation] = cs + } } if err := controllerutil.SetControllerReference(broker, pod, scheme); err != nil { return ctrl.Result{}, err From 43b019f95fb8ab77f163a925609b5590cd85d9db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:03:14 +0200 Subject: [PATCH 11/29] fixup! operator: add the Broker CR reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M6: resolveBrokerID only matched the per-pod-FQDN advertised-address form, so host:port / bare-host / pod-IP advertise modes never resolved an ID — registration stayed unverified and decommission stalled. Rewritten as resolveBroker: port-stripping, first-label / raw-host / pod-IP matching with ambiguity refusal (mirroring the v2 controller), returning the full membership entry. Identity ADOPTION additionally requires an active+alive entry: after a decommission the membership list can briefly retain the dead predecessor under the same reused pod name, and adopting that id poisons the node_id continuity check permanently. --- .../controller/redpanda/broker_controller.go | 72 ++++++++++++++++--- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index 16deae561a..6f3ecc6fac 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -12,11 +12,13 @@ package redpanda import ( "context" "fmt" + "net" "strings" "time" "github.com/go-logr/logr" "github.com/redpanda-data/common-go/otelutil/log" + "github.com/redpanda-data/common-go/rpadmin" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -436,12 +438,12 @@ func (r *BrokerReconciler) reconcileBrokerRegistration(ctx context.Context, stat l := log.FromContext(ctx) podName := broker.PodName() - currentID, err := r.resolveBrokerID(ctx, state.clusterName, broker, podName) + resolved, err := r.resolveBroker(ctx, state.clusterName, broker, state.pod, podName) if err != nil { l.Info("could not resolve broker ID, will retry", "error", err) return ctrl.Result{RequeueAfter: requeueShort}, nil } - if currentID == nil { + if resolved == nil { if broker.Status.BrokerID != nil { // Was registered, no longer a member (e.g. removed out of // band): report unverified but let the chain continue. @@ -451,8 +453,18 @@ func (r *BrokerReconciler) reconcileBrokerRegistration(ctx context.Context, stat // Pod is up but not yet a cluster member. return ctrl.Result{RequeueAfter: requeueShort}, nil } + currentID := ptr.To(int32(resolved.NodeID)) if broker.Status.BrokerID == nil { + // Adopt only an active, alive member. Right after a decommission the + // membership list can briefly retain the dead predecessor's entry + // under this very pod name — adopting its id would poison the + // continuity check below for the rest of this Broker's life. + if !brokerActiveAndAlive(resolved) { + l.Info("matched membership entry not active/alive yet, deferring identity adoption", + "nodeID", resolved.NodeID, "membership", resolved.MembershipStatus) + return ctrl.Result{RequeueAfter: requeueShort}, nil + } broker.Status.BrokerID = currentID } if *currentID != *broker.Status.BrokerID { @@ -499,15 +511,15 @@ func (r *BrokerReconciler) reconcileDecommission(ctx context.Context, state *bro // or an intent set before the first registration (or right after a // status-update race) never actually starts the decommission and the // broker stays a full cluster member while reporting Decommissioning. - currentID, err := r.resolveBrokerID(ctx, state.clusterName, broker, broker.PodName()) - if err != nil || currentID == nil { + resolved, err := r.resolveBroker(ctx, state.clusterName, broker, state.pod, broker.PodName()) + if err != nil || resolved == nil || resolved.MembershipStatus != rpadmin.MembershipStatusActive { if err != nil { log.FromContext(ctx).Info("could not resolve broker ID before decommission, will retry", "error", err) } state.phase = redpandav1alpha2.BrokerPhaseDecommissioning return ctrl.Result{RequeueAfter: requeueShort}, nil } - broker.Status.BrokerID = currentID + broker.Status.BrokerID = ptr.To(int32(resolved.NodeID)) } decommResult, err := r.executeDecommission(ctx, state.clusterName, broker) @@ -824,12 +836,14 @@ func (r *BrokerReconciler) reconcileDelete(ctx context.Context, l logr.Logger, k // it live rather than skipping the decommission and leaving a dead // membership entry behind. if broker.Status.BrokerID == nil { - brokerID, err := r.resolveBrokerID(ctx, clusterName, broker, podName) + resolved, err := r.resolveBroker(ctx, clusterName, broker, &pod, podName) if err != nil { l.Info("could not resolve broker ID before decommission, will retry", "error", err) return ctrl.Result{RequeueAfter: requeueShort}, nil } - broker.Status.BrokerID = brokerID + if resolved != nil && resolved.MembershipStatus == rpadmin.MembershipStatusActive { + broker.Status.BrokerID = ptr.To(int32(resolved.NodeID)) + } } if broker.Status.BrokerID != nil { result, err := r.executeDecommission(ctx, clusterName, broker) @@ -1089,7 +1103,16 @@ func (r *BrokerReconciler) disableMaintenanceMode(ctx context.Context, clusterNa return admin.DisableMaintenanceMode(ctx, int(*broker.Status.BrokerID), false) } -func (r *BrokerReconciler) resolveBrokerID(ctx context.Context, clusterName string, broker *redpandav1alpha2.Broker, podName string) (*int32, error) { +// resolveBroker finds the cluster-membership entry backing this Broker's +// pod. Matching handles every advertised-address form seen in the wild: +// per-pod FQDN (matched by first DNS label), "host:port", bare host, and — +// when the pod is known — a bare pod IP. More than one distinct match is +// ambiguous and reported as no match rather than guessed at: right after a +// decommission the membership list can briefly retain the dead +// predecessor's entry under the SAME pod name (a replacement reuses it). +// Callers that ADOPT the resolved id must additionally check the entry is +// an active, alive member — see reconcileBrokerRegistration. +func (r *BrokerReconciler) resolveBroker(ctx context.Context, clusterName string, broker *redpandav1alpha2.Broker, pod *corev1.Pod, podName string) (*rpadmin.Broker, error) { admin, err := r.ClientFactory.RedpandaAdminClientForCluster(ctx, broker, clusterName) if err != nil { return nil, err @@ -1100,12 +1123,39 @@ func (r *BrokerReconciler) resolveBrokerID(ctx context.Context, clusterName stri if err != nil { return nil, err } + + var matches []rpadmin.Broker for _, b := range brokers { - if strings.Split(b.InternalRPCAddress, ".")[0] == podName { - return ptr.To(int32(b.NodeID)), nil + host := b.InternalRPCAddress + if h, _, splitErr := net.SplitHostPort(host); splitErr == nil { + host = h + } + if host == "" { + continue + } + if strings.SplitN(host, ".", 2)[0] == podName || host == podName || + (pod != nil && pod.Status.PodIP != "" && host == pod.Status.PodIP) { + matches = append(matches, b) } } - return nil, nil + switch len(matches) { + case 0: + return nil, nil + case 1: + return &matches[0], nil + default: + log.FromContext(ctx).Info("ambiguous cluster-membership match for pod, refusing to guess", + "pod", podName, "matches", len(matches)) + return nil, nil + } +} + +// brokerActiveAndAlive reports whether a membership entry is safe to adopt +// as this Broker's identity: an entry that is draining, removed, or not +// alive is either a leftover of a decommissioned predecessor or a node that +// has not finished joining. +func brokerActiveAndAlive(b *rpadmin.Broker) bool { + return b != nil && b.MembershipStatus == rpadmin.MembershipStatusActive && b.IsAlive != nil && *b.IsAlive } // hasValidRollGrant returns true if the Broker CR carries a roll-grant From aceb3ca94ac02ddaa4e238c7d52fd024f2a6798d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:09:27 +0200 Subject: [PATCH 12/29] fixup! operator: add Broker CRD API, conditions, and roll-grant feature flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M7: PodName had two unvalidated hazards. (a) networkIndex was optional and nil defaulted to 0, so a Broker omitting it collides with an explicit networkIndex: 0 on the same pod name — the field is now required with Minimum=0 (CRD regenerated). (b) A NodePool-referenced Broker missing the cluster-name label produced "--" — an invalid DNS name rejected on every pod create in an error loop; the reconciler now surfaces that as Stuck until the label convention gets CEL/webhook enforcement. --- operator/api/redpanda/v1alpha2/broker_types.go | 7 +++++-- .../redpanda/v1alpha2/testdata/crd-docs.adoc | 5 ++++- .../crd/bases/cluster.redpanda.com_brokers.yaml | 4 ++++ .../controller/redpanda/broker_controller.go | 17 +++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/operator/api/redpanda/v1alpha2/broker_types.go b/operator/api/redpanda/v1alpha2/broker_types.go index 2f7d8efcc9..5e6847ed0c 100644 --- a/operator/api/redpanda/v1alpha2/broker_types.go +++ b/operator/api/redpanda/v1alpha2/broker_types.go @@ -120,8 +120,11 @@ type BrokerSpec struct { // For ordinal-named pods this is the pod ordinal; the V2 chart uses it // to select external.addresses[networkIndex]. The value is captured from // the parsed ordinal at migration time and preserved across pod rotation. - // +optional - NetworkIndex *int32 `json:"networkIndex,omitempty"` + // Required: the index is also the pod-name suffix, so an omitted value + // would collide with an explicit networkIndex: 0. + // +required + // +kubebuilder:validation:Minimum=0 + NetworkIndex *int32 `json:"networkIndex"` // PodTemplate is the fully resolved pod spec for this broker. PodTemplate BrokerPodTemplate `json:"podTemplate"` diff --git a/operator/api/redpanda/v1alpha2/testdata/crd-docs.adoc b/operator/api/redpanda/v1alpha2/testdata/crd-docs.adoc index 4294a18ee0..d43c774822 100644 --- a/operator/api/redpanda/v1alpha2/testdata/crd-docs.adoc +++ b/operator/api/redpanda/v1alpha2/testdata/crd-docs.adoc @@ -683,7 +683,10 @@ explicitly set. + | | Optional: \{} + | *`networkIndex`* __integer__ | NetworkIndex is the stable external-addressing slot for this broker. + For ordinal-named pods this is the pod ordinal; the V2 chart uses it + to select external.addresses[networkIndex]. The value is captured from + -the parsed ordinal at migration time and preserved across pod rotation. + | | Optional: \{} + +the parsed ordinal at migration time and preserved across pod rotation. + +Required: the index is also the pod-name suffix, so an omitted value + +would collide with an explicit networkIndex: 0. + | | Minimum: 0 + +Required: \{} + | *`podTemplate`* __xref:{anchor_prefix}-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-brokerpodtemplate[$$BrokerPodTemplate$$]__ | PodTemplate is the fully resolved pod spec for this broker. + | | | *`storage`* __xref:{anchor_prefix}-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-brokerstorage[$$BrokerStorage$$]__ | Storage defines PVC templates or references to existing claims. + | | diff --git a/operator/config/crd/bases/cluster.redpanda.com_brokers.yaml b/operator/config/crd/bases/cluster.redpanda.com_brokers.yaml index 6e2bd9c2dc..06ee75fca8 100644 --- a/operator/config/crd/bases/cluster.redpanda.com_brokers.yaml +++ b/operator/config/crd/bases/cluster.redpanda.com_brokers.yaml @@ -101,7 +101,10 @@ spec: For ordinal-named pods this is the pod ordinal; the V2 chart uses it to select external.addresses[networkIndex]. The value is captured from the parsed ordinal at migration time and preserved across pod rotation. + Required: the index is also the pod-name suffix, so an omitted value + would collide with an explicit networkIndex: 0. format: int32 + minimum: 0 type: integer podTemplate: description: PodTemplate is the fully resolved pod spec for this broker. @@ -8820,6 +8823,7 @@ spec: type: object required: - clusterRef + - networkIndex - podTemplate - storage type: object diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index 6f3ecc6fac..21bb704aac 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -164,6 +164,23 @@ func (r *BrokerReconciler) Reconcile(ctx context.Context, req mcreconcile.Reques return r.reconcileDelete(ctx, l, k8sClient, req.ClusterName, &broker, broker.PodName()) } + // A NodePool-referenced Broker without the cluster-name label cannot + // derive its pod name — it would come out as "--", an + // invalid DNS name rejected on every pod create. Surface Stuck instead + // of error-looping. (The label is convention until CEL/webhook + // validation exists.) + if broker.Spec.ClusterRef.IsNodePool() && broker.Labels[redpandav1alpha2.ClusterNameLabel] == "" { + l.Info("NodePool-referenced Broker is missing the cluster-name label; cannot derive its pod name", + "label", redpandav1alpha2.ClusterNameLabel) + state := &brokerReconciliationState{ + broker: &broker, + phase: redpandav1alpha2.BrokerPhaseStuck, + clusterName: req.ClusterName, + initialStatus: broker.Status.DeepCopy(), + } + return r.syncBrokerStatus(ctx, state, k8sCluster, ctrl.Result{RequeueAfter: periodicRequeue}) + } + state, err := r.fetchState(ctx, k8sClient, &broker) if err != nil { return ctrl.Result{}, err From b58dd5cafec42bd1f9e0396ced2b01fecbf9158a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:11:04 +0200 Subject: [PATCH 13/29] fixup! operator: add the Broker CR reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M8: the changelog advertises dead-node PV remediation as a Broker capability, but remediatePVAffinity silently no-ops at the default --unbind-pvcs-after=0 and the flag help described it as PVCUnbinder-only — an operator enabling --enable-broker alone gets a permanently Stuck broker with no hint. The flag help, the disabled-path log line, and the changelog entry now name the prerequisite. --- .changes/unreleased/operator-Added-20260709-120000.yaml | 2 +- operator/cmd/run/run.go | 2 +- operator/internal/controller/redpanda/broker_controller.go | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.changes/unreleased/operator-Added-20260709-120000.yaml b/.changes/unreleased/operator-Added-20260709-120000.yaml index cb5765076e..86a807d5ff 100644 --- a/.changes/unreleased/operator-Added-20260709-120000.yaml +++ b/.changes/unreleased/operator-Added-20260709-120000.yaml @@ -1,4 +1,4 @@ project: operator kind: Added -body: Broker custom resource and reconciler (experimental, behind --enable-broker) — per-broker pod/PVC lifecycle with grant-gated serialized rotation, leadership drain, node_id continuity verification, explicit-intent decommission with recommission and last-broker guard, dead-node PV remediation, and full status conditions +body: Broker custom resource and reconciler (experimental, behind --enable-broker) — per-broker pod/PVC lifecycle with grant-gated serialized rotation, leadership drain, node_id continuity verification, explicit-intent decommission with recommission and last-broker guard, dead-node PV remediation (requires --unbind-pvcs-after to be set), and full status conditions time: 2026-07-09T12:00:00.000000000+02:00 diff --git a/operator/cmd/run/run.go b/operator/cmd/run/run.go index 36063136f3..2b9c986292 100644 --- a/operator/cmd/run/run.go +++ b/operator/cmd/run/run.go @@ -201,7 +201,7 @@ func (o *RunOptions) BindFlags(cmd *cobra.Command) { cmd.Flags().BoolVar(&o.ghostbuster, "unsafe-decommission-failed-brokers", false, "Set to enable decommissioning a failed broker that is configured but does not exist in the StatefulSet (ghost broker). This may result in invalidating valid data") _ = cmd.Flags().MarkHidden("unsafe-decommission-failed-brokers") cmd.Flags().StringSliceVar(&o.additionalControllers, "additional-controllers", []string{""}, fmt.Sprintf("which controllers to run, available: all, %s", strings.Join(availableControllers, ", "))) - cmd.Flags().DurationVar(&o.unbindPVCsAfter, "unbind-pvcs-after", 0, "if not zero, runs the PVCUnbinder controller which attempts to 'unbind' the PVCs' of Pods that are Pending for longer than the given duration") + cmd.Flags().DurationVar(&o.unbindPVCsAfter, "unbind-pvcs-after", 0, "if not zero, runs the PVCUnbinder controller which attempts to 'unbind' the PVCs' of Pods that are Pending for longer than the given duration; also a prerequisite for the Broker controller's dead-node PV remediation (--enable-broker), which stays disabled while this is zero") cmd.Flags().BoolVar(&o.allowPVRebinding, "allow-pv-rebinding", false, "DEPRECATED. When the PVCUnbinder fires, also clear the freed PV's ClaimRef so the disk can be reused if the node returns. Risks cross-broker disk swap when multiple PVs are cleared concurrently. Leave at false (the default) — the unbinder's pause-annotation, multi-pod auto-detect, and per-cluster serialization gates make this flag unnecessary in practice and unsafe in the cases where it would have been useful.") _ = cmd.Flags().MarkDeprecated("allow-pv-rebinding", "the gating checks added to the PVCUnbinder (pause annotation, multi-pod auto-detect, per-cluster serialization) supersede this flag; see the PVCUnbinder controller godoc") cmd.Flags().Var(&o.unbinderSelector, "unbinder-label-selector", "if provided, a Kubernetes label selector that will filter Pods to be considered by the PVCUnbinder.") diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index 21bb704aac..a9cb095988 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -1056,7 +1056,9 @@ func (r *BrokerReconciler) ensureDrained(ctx context.Context, clusterName string // Returns false if the timeout hasn't elapsed or the node still exists. func (r *BrokerReconciler) remediatePVAffinity(ctx context.Context, l logr.Logger, k8sClient client.Client, apiReader client.Reader, broker *redpandav1alpha2.Broker, pod *corev1.Pod) (bool, error) { if r.UnbindPVCsAfter <= 0 { - l.Info("PV affinity remediation disabled (unbind-pvcs-after=0)") + l.Info("broker pod is stuck on PV node affinity but remediation is disabled; " + + "set --unbind-pvcs-after to a non-zero duration to enable dead-node PV remediation " + + "(the broker will otherwise stay Stuck until the node returns or the PVC is removed manually)") return false, nil } From 1f5caaa303395c861bb68e61877852274ff15588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:14:54 +0200 Subject: [PATCH 14/29] fixup! operator: add Broker controller integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M9 (integration half): the roll-grant gate had no negative coverage — setupBrokerCluster blanket-granted every broker, and TestPodRotationWithoutGrant only asserted ConfigSynced=False, which is set BEFORE a rotation completes, so a gate that always allowed rotation stayed green. Grants are now opt-in in the fixture; the rotation test asserts the ungranted outdated pod keeps its UID (require.Never), then grants and asserts the rotation completes; TestPVAffinityRemediation starts ungranted so its Stuck wait proves remediation refusal. --- .../redpanda/broker_controller_test.go | 81 ++++++++++++++----- 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller_test.go b/operator/internal/controller/redpanda/broker_controller_test.go index 8ed0b98da1..4c87af8e97 100644 --- a/operator/internal/controller/redpanda/broker_controller_test.go +++ b/operator/internal/controller/redpanda/broker_controller_test.go @@ -461,27 +461,33 @@ func (s *BrokerControllerSuite) TestPodCreatedWithoutGrant() { }, 2*time.Minute, time.Second, "pod should be created without a roll-grant") } -// TestPodRotationWithoutGrant verifies that when a broker's pod needs rotation -// (checksum mismatch) but no roll-grant is present, the broker stays in Running -// phase and sets ConfigSynced=False. +// TestPodRotationWithoutGrant verifies the roll-grant gate NEGATIVELY: an +// outdated pod with NO grant at all must not rotate — the pod's UID and live +// checksum stay put while ConfigSynced=False reports the pending rotation. +// Only after a matching grant is issued may the pod be recreated. Without +// the negative half, a gate that always allowed rotation would be +// observationally identical to a working one (ConfigSynced=False is set as +// soon as drift is detected, BEFORE the rotation completes, so an Eventually +// on it alone can pass mid-rotation). func (s *BrokerControllerSuite) TestPodRotationWithoutGrant() { t, ctx, cancel, c := s.setup() defer cancel() - _, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ - grantDuration: 10 * time.Minute, - }) + _, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{}) target := brokers[0] require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) - // Change the desired checksum so it no longer matches the pod's checksum - // (and no longer matches the existing roll-grant's checksum). + var podBefore corev1.Pod + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: target.PodName(), Namespace: target.Namespace}, &podBefore)) + + // Change the desired checksum so the pod is outdated; no grant exists. + newChecksum := "deliberately-changed-checksum" p := client.MergeFrom(target.DeepCopy()) - target.Spec.PodTemplate.Annotations["config.redpanda.com/checksum"] = "deliberately-wrong-checksum" + target.Spec.PodTemplate.Annotations["config.redpanda.com/checksum"] = newChecksum require.NoError(t, c.Patch(ctx, target, p)) - // The broker should stay Running (not rotated) with ConfigSynced=False. + // Drift is reported... require.EventuallyWithT(t, func(ct *assert.CollectT) { assert.NoError(ct, c.Get(ctx, client.ObjectKeyFromObject(target), target)) assert.Equal(ct, redpandav1alpha2.BrokerPhaseRunning, target.Status.Phase) @@ -490,6 +496,33 @@ func (s *BrokerControllerSuite) TestPodRotationWithoutGrant() { assert.Equal(ct, metav1.ConditionFalse, cond.Status) } }, 2*time.Minute, 5*time.Second) + + // ...but the pod must NOT be rotated: same UID, same live checksum. + require.Never(t, func() bool { + var pod corev1.Pod + if err := c.Get(ctx, client.ObjectKey{Name: target.PodName(), Namespace: target.Namespace}, &pod); err != nil { + return true // pod deleted = rotation started + } + return pod.UID != podBefore.UID + }, 30*time.Second, 2*time.Second, "pod was rotated without a roll-grant") + + // Issue a grant matching the new checksum: the rotation may now proceed. + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(target), target)) + p = client.MergeFrom(target.DeepCopy()) + if target.Annotations == nil { + target.Annotations = map[string]string{} + } + target.Annotations["operator.redpanda.com/roll-grant"] = newChecksum + "/" + strconv.FormatInt(time.Now().Add(10*time.Minute).Unix(), 10) + require.NoError(t, c.Patch(ctx, target, p)) + + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var pod corev1.Pod + if !assert.NoError(ct, c.Get(ctx, client.ObjectKey{Name: target.PodName(), Namespace: target.Namespace}, &pod)) { + return + } + assert.NotEqual(ct, podBefore.UID, pod.UID, "pod should be recreated once granted") + assert.Equal(ct, newChecksum, pod.Annotations["config.redpanda.com/checksum"]) + }, 4*time.Minute, 5*time.Second, "granted rotation never completed") } // TestLastBrokerDecommissionGuard verifies that attempting to decommission the @@ -523,9 +556,10 @@ func (s *BrokerControllerSuite) TestPVAffinityRemediation() { t, ctx, cancel, c := s.setupSerial() defer cancel() + // No setup grant: the Stuck wait below must prove the ungranted broker + // REFUSES to remediate; the explicit grant later is what unlocks it. _, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ useVolumeClaimTemplates: true, - grantDuration: 30 * time.Minute, }) // Pick a target broker whose pod runs on an AGENT node: k3d server nodes @@ -779,16 +813,23 @@ func (s *BrokerControllerSuite) setupBrokerCluster(t *testing.T, ctx context.Con brokers = append(brokers, broker) } - deadline := strconv.FormatInt(time.Now().Add(opts.grantDuration).Unix(), 10) - for _, b := range brokers { - require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(b), b)) - p := client.MergeFrom(b.DeepCopy()) - if b.Annotations == nil { - b.Annotations = map[string]string{} + // Grants are opt-in: pod creation and adoption are deliberately NOT + // grant-gated (RFC Q5), so the steady state needs none. Tests that + // exercise disruptive actions grant explicitly — a blanket grant here + // would make it impossible to assert that ungranted brokers refuse to + // rotate or remediate. + if opts.grantDuration > 0 { + deadline := strconv.FormatInt(time.Now().Add(opts.grantDuration).Unix(), 10) + for _, b := range brokers { + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(b), b)) + p := client.MergeFrom(b.DeepCopy()) + if b.Annotations == nil { + b.Annotations = map[string]string{} + } + checksum := b.Spec.PodTemplate.Annotations["config.redpanda.com/checksum"] + b.Annotations["operator.redpanda.com/roll-grant"] = checksum + "/" + deadline + require.NoError(t, c.Patch(ctx, b, p)) } - checksum := b.Spec.PodTemplate.Annotations["config.redpanda.com/checksum"] - b.Annotations["operator.redpanda.com/roll-grant"] = checksum + "/" + deadline - require.NoError(t, c.Patch(ctx, b, p)) } for _, b := range brokers { From d408e720a3e148ceed3ce60b0d55b2048a7d7360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:14:54 +0200 Subject: [PATCH 15/29] fixup! operator: add Broker CR acceptance coverage M9 (acceptance half): the adoption scenario granted the only outdated broker immediately, so "outdated+granted rotates" and "gate ignored, rotates anyway" were observationally identical. A new step asserts the outdated pod keeps its UID for a fixed window BEFORE the grant is issued. --- .../features/broker-crd-adoption.feature | 6 +++-- acceptance/steps/broker.go | 25 +++++++++++++++++++ acceptance/steps/register.go | 1 + 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/acceptance/features/broker-crd-adoption.feature b/acceptance/features/broker-crd-adoption.feature index 66dbc39d47..b8533bf0d0 100644 --- a/acceptance/features/broker-crd-adoption.feature +++ b/acceptance/features/broker-crd-adoption.feature @@ -11,8 +11,10 @@ Feature: Broker CRD pod adoption Then all Broker CRs for cluster "broker-test" should be Running And cluster "broker-test" admin API should show 3 brokers When I update Broker "broker-test-0" pod template with env "ROTATION_TEST=applied" for cluster "broker-test" - # Rotation IS gated: only the targeted broker gets a grant. - And I grant a roll-grant to Broker "broker-test-0" + # Rotation IS gated: an outdated pod without a grant must stay put... + Then Broker "broker-test-0" pod should not be rotated + # ...and only once granted may the rotation proceed. + When I grant a roll-grant to Broker "broker-test-0" Then Broker "broker-test-0" pod should have env "ROTATION_TEST" = "applied" And all Broker CRs for cluster "broker-test" should be Running When I set decommission on Broker "broker-test-2" diff --git a/acceptance/steps/broker.go b/acceptance/steps/broker.go index 8b076d8930..ccb232bd82 100644 --- a/acceptance/steps/broker.go +++ b/acceptance/steps/broker.go @@ -211,6 +211,31 @@ func updateBrokerPodTemplateEnv(ctx context.Context, t framework.TestingT, broke t.Logf("Updated Broker %q pod template: added env %s, new checksum %s", brokerName, envKeyValue, checksum[:12]) } +// brokerPodShouldNotBeRotated asserts for a fixed window that the Broker's +// pod is neither deleted nor recreated — the negative half of the roll-grant +// gate: an outdated pod WITHOUT a grant must stay put. Without this check, a +// gate that always allowed rotation would be observationally identical to a +// working one. +func brokerPodShouldNotBeRotated(ctx context.Context, t framework.TestingT, brokerName string) { + key := t.ResourceKey(brokerName) + var broker redpandav1alpha2.Broker + require.NoError(t, t.Get(ctx, key, &broker)) + + podKey := runtimeclient.ObjectKey{Name: broker.PodName(), Namespace: key.Namespace} + var pod corev1.Pod + require.NoError(t, t.Get(ctx, podKey, &pod)) + originalUID := pod.UID + + require.Never(t, func() bool { + var current corev1.Pod + if err := t.Get(ctx, podKey, ¤t); err != nil { + return true // pod deleted = rotation started + } + return current.UID != originalUID + }, 30*time.Second, 2*time.Second, "pod %q was rotated without a roll-grant", broker.PodName()) + t.Logf("Pod %q kept UID %s without a roll-grant", broker.PodName(), originalUID) +} + func brokerPodShouldHaveEnv(ctx context.Context, t framework.TestingT, brokerName, envName, envValue string) { key := t.ResourceKey(brokerName) require.Eventually(t, func() bool { diff --git a/acceptance/steps/register.go b/acceptance/steps/register.go index e5ca8821c3..5726d561b4 100644 --- a/acceptance/steps/register.go +++ b/acceptance/steps/register.go @@ -187,6 +187,7 @@ func init() { framework.RegisterStep(`^cluster "([^"]*)" admin API should show (\d+) brokers$`, clusterAdminAPIShouldShowBrokers) framework.RegisterStep(`^I update Broker "([^"]*)" pod template with env "([^"]*)" for cluster "([^"]*)"$`, updateBrokerPodTemplateEnv) framework.RegisterStep(`^Broker "([^"]*)" pod should have env "([^"]*)" = "([^"]*)"$`, brokerPodShouldHaveEnv) + framework.RegisterStep(`^Broker "([^"]*)" pod should not be rotated$`, brokerPodShouldNotBeRotated) // Debug steps framework.RegisterStep(`^I become debuggable$`, sleepALongTime) From 981c48240e005b1224f0c72eb3cc5b795f7219e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:18:29 +0200 Subject: [PATCH 16/29] fixup! operator: add Broker controller integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M10: TestPVAffinityRemediation deleted a node on the SHARED k3d cluster — serializing within this suite is not enough because test packages run concurrently and other packages testenvs share that cluster; their pods and node-pinned PVCs would be stranded (pkg/k3d explicitly warns against this). The test now runs on a dedicated k3d cluster (env construction factored into newEnv with a cluster-name option, following the pvcunbinder pattern), which also let the replacement-node restoration cleanup be dropped. Additionally, the replacement node created mid-test now gets the operator and Redpanda images re-imported — k3d node create starts empty and soft anti-affinity prefers the new node, so the sidecar would ImagePullBackOff and the recovery wait would time out. --- .../redpanda/broker_controller_test.go | 110 ++++++++---------- 1 file changed, 47 insertions(+), 63 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller_test.go b/operator/internal/controller/redpanda/broker_controller_test.go index 4c87af8e97..2fdab38505 100644 --- a/operator/internal/controller/redpanda/broker_controller_test.go +++ b/operator/internal/controller/redpanda/broker_controller_test.go @@ -69,13 +69,6 @@ func (s *BrokerControllerSuite) setup() (*testing.T, context.Context, context.Ca return s.setupNamespace(t) } -// setupSerial is setup without t.Parallel, for tests that disrupt shared -// cluster infrastructure (e.g. deleting k3d nodes): running those alongside -// parallel tests strands the other tests' pods and PVCs on the dead node. -func (s *BrokerControllerSuite) setupSerial() (*testing.T, context.Context, context.CancelFunc, client.Client) { - return s.setupNamespace(s.T()) -} - func (s *BrokerControllerSuite) setupNamespace(t *testing.T) (*testing.T, context.Context, context.CancelFunc, client.Client) { ctx, cancel := context.WithTimeout(trace.Test(t), 15*time.Minute) ns := s.env.CreateTestNamespace(t) @@ -84,6 +77,15 @@ func (s *BrokerControllerSuite) setupNamespace(t *testing.T) (*testing.T, contex func (s *BrokerControllerSuite) SetupSuite() { t := s.T() + s.env, s.clientFactory = s.newEnv(t, "") +} + +// newEnv builds a testenv (shared k3d cluster when clusterName is empty, a +// dedicated one otherwise) with the Broker/Redpanda/NodePool controllers and +// RBAC set up. Tests that disrupt cluster infrastructure (node deletion) use +// a dedicated cluster so concurrently-running test PACKAGES sharing the +// default cluster don't lose pods and node-pinned PVCs. +func (s *BrokerControllerSuite) newEnv(t *testing.T, clusterName string) (*testenv.Env, internalclient.ClientFactory) { ctx := trace.Test(t) importImages := []string{ @@ -102,7 +104,8 @@ func (s *BrokerControllerSuite) SetupSuite() { } } - s.env = testenv.New(t, testenv.Options{ + env := testenv.New(t, testenv.Options{ + Name: clusterName, Scheme: controller.V2Scheme, CRDs: crds.All(), Logger: log.FromContext(ctx), @@ -111,11 +114,11 @@ func (s *BrokerControllerSuite) SetupSuite() { ImportImages: importImages, }) - suiteClient := s.env.Client() + var clientFactory internalclient.ClientFactory - s.env.SetupManager(s.setupRBAC(ctx, suiteClient), func(mgr multicluster.Manager) error { + env.SetupManager(s.setupRBAC(ctx, env), func(mgr multicluster.Manager) error { dialer := kube.NewPodDialer(mgr.GetLocalManager().GetConfig()) - s.clientFactory = internalclient.NewFactory(mgr, nil).WithDialer(dialer.DialContext) + clientFactory = internalclient.NewFactory(mgr, nil).WithDialer(dialer.DialContext) require.NoError(t, (&redpanda.NodePoolReconciler{ Manager: mgr, @@ -123,7 +126,7 @@ func (s *BrokerControllerSuite) SetupSuite() { require.NoError(t, (&redpanda.RedpandaReconciler{ Manager: mgr, - ClientFactory: s.clientFactory, + ClientFactory: clientFactory, LifecycleClient: lifecycle.NewResourceClient(mgr, lifecycle.V2ResourceManagers( lifecycle.Image{Repository: os.Getenv("TEST_REDPANDA_REPO"), Tag: os.Getenv("TEST_REDPANDA_VERSION")}, lifecycle.Image{Repository: "localhost/redpanda-operator", Tag: "dev"}, @@ -132,12 +135,15 @@ func (s *BrokerControllerSuite) SetupSuite() { UseNodePools: true, }).SetupWithManager(ctx, mgr, "")) - return redpanda.SetupBrokerController(ctx, mgr, s.clientFactory, "", 60*time.Second) + return redpanda.SetupBrokerController(ctx, mgr, clientFactory, "", 60*time.Second) }) + + return env, clientFactory } -func (s *BrokerControllerSuite) setupRBAC(ctx context.Context, c client.Client) string { +func (s *BrokerControllerSuite) setupRBAC(ctx context.Context, env *testenv.Env) string { t := s.T() + c := env.Client() roles, err := kube.DecodeYAML(operatorRBAC, c.Scheme()) require.NoError(t, err) @@ -171,7 +177,7 @@ func (s *BrokerControllerSuite) setupRBAC(ctx context.Context, c client.Client) Name: name, }, Subjects: []rbacv1.Subject{ - {Kind: "ServiceAccount", Namespace: s.env.Namespace(), Name: name}, + {Kind: "ServiceAccount", Namespace: env.Namespace(), Name: name}, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", @@ -551,11 +557,19 @@ func (s *BrokerControllerSuite) TestLastBrokerDecommissionGuard() { // force-delete the stuck pod, and assert the Broker controller reports // Stuck then — once granted a roll — remediates and recovers. func (s *BrokerControllerSuite) TestPVAffinityRemediation() { - // Serial: this test deletes a shared k3d node; parallel neighbors would - // lose pods and node-pinned PVCs with it. - t, ctx, cancel, c := s.setupSerial() + t := s.T() + t.Parallel() + ctx, cancel := context.WithTimeout(trace.Test(t), 20*time.Minute) defer cancel() + // Dedicated k3d cluster: this test deletes a node. Serializing within + // this suite is not enough — test PACKAGES run concurrently and other + // packages' testenvs share the default cluster; their pods and + // node-pinned PVCs would be stranded by the deletion. + env, _ := s.newEnv(t, "broker-pv-"+strings.ToLower(testenv.RandString(4))) + ns := env.CreateTestNamespace(t) + c := ns.Client + // No setup grant: the Stuck wait below must prove the ungranted broker // REFUSES to remediate; the explicit grant later is what unlocks it. _, brokers := s.setupBrokerCluster(t, ctx, c, brokerClusterOpts{ @@ -598,50 +612,10 @@ func (s *BrokerControllerSuite) TestPVAffinityRemediation() { originalPVNames[name] = pvc.Spec.VolumeName } - // Delete the k3d node. Ensure a replacement is always created, even on - // test failure, so other tests sharing this k3d cluster are not impacted. - var nodesBefore corev1.NodeList - require.NoError(t, s.env.Client().List(ctx, &nodesBefore)) - expectedNodeCount := len(nodesBefore.Items) - + // Delete the k3d node. No restoration cleanup is needed: the dedicated + // cluster is torn down with the env at test end. t.Logf("deleting k3d node %q", targetNode) - require.NoError(t, s.env.Host().DeleteNode(targetNode)) - t.Cleanup(func() { - ctx := context.Background() - // Remove the stale Node object if the test failed before doing so - // itself: a dead node lingering in the API would both skew the count - // below and confuse later tests. - var staleNode corev1.Node - if err := s.env.Client().Get(ctx, client.ObjectKey{Name: targetNode}, &staleNode); err == nil { - if err := s.env.Client().Delete(ctx, &staleNode); err != nil { - t.Logf("WARNING: failed to delete stale Node object %q during cleanup: %v", targetNode, err) - } - } - var nodes corev1.NodeList - if err := s.env.Client().List(ctx, &nodes); err != nil { - t.Logf("WARNING: failed to list nodes during cleanup: %v", err) - return - } - ready := 0 - for _, node := range nodes.Items { - if node.Name == targetNode { - continue - } - for _, cond := range node.Status.Conditions { - if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue { - ready++ - break - } - } - } - if ready >= expectedNodeCount { - return - } - t.Logf("cluster has %d ready nodes (expected %d), creating replacement", ready, expectedNodeCount) - if err := s.env.Host().CreateNode(); err != nil { - t.Logf("WARNING: failed to create replacement k3d node during cleanup: %v", err) - } - }) + require.NoError(t, env.Host().DeleteNode(targetNode)) // Delete the Kubernetes Node object so the Broker controller sees it as gone. var nodeObj corev1.Node @@ -679,9 +653,19 @@ func (s *BrokerControllerSuite) TestPVAffinityRemediation() { target.Annotations["operator.redpanda.com/roll-grant"] = checksum + "/" + strconv.FormatInt(time.Now().Add(30*time.Minute).Unix(), 10) require.NoError(t, c.Patch(ctx, target, p)) - // Create a replacement k3d node so the pod has somewhere to schedule. + // Create a replacement k3d node so the pod has somewhere to schedule, + // and re-import the suite's images: `k3d node create` starts an empty + // node, and soft anti-affinity prefers it — without the operator image + // present the sidecar would sit in ImagePullBackOff and the recovery + // wait below would time out. t.Log("creating replacement k3d node") - require.NoError(t, s.env.Host().CreateNode()) + require.NoError(t, env.Host().CreateNode()) + require.NoError(t, env.Host().ImportImage("localhost/redpanda-operator:dev")) + if repo := os.Getenv("TEST_REDPANDA_REPO"); repo != "" { + if version := os.Getenv("TEST_REDPANDA_VERSION"); version != "" { + require.NoError(t, env.Host().ImportImage(fmt.Sprintf("%s:%s", repo, version))) + } + } // Wait for the broker to recover to Running. t.Log("waiting for Broker to recover to Running") From 108e5d8f6de003745f79fb634aabf6e5c33181af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:19:47 +0200 Subject: [PATCH 17/29] fixup! operator: add Broker CR acceptance coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M11: (a) createBrokerCRsForCluster copied the live pod spec verbatim into the Broker pod template — including NodeName, which pins every replacement pod to the original node; if that node is gone, scheduling times out. The node name is now cleared, mirroring the integration helper. (b) orphanDeleteStatefulSet did not wait for the deletion to settle, so a racing pre-pause reconcile could recreate the StatefulSet and re-adopt the pods, corrupting the broker-count assertions; the step now fails fast if the STS does not stay gone. --- acceptance/steps/broker.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/acceptance/steps/broker.go b/acceptance/steps/broker.go index ccb232bd82..deb131a3b9 100644 --- a/acceptance/steps/broker.go +++ b/acceptance/steps/broker.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -49,6 +50,15 @@ func orphanDeleteStatefulSet(ctx context.Context, t framework.TestingT, clusterN var sts appsv1.StatefulSet require.NoError(t, t.Get(ctx, key, &sts)) require.NoError(t, t.Delete(ctx, &sts, runtimeclient.PropagationPolicy(metav1.DeletePropagationOrphan))) + + // The Redpanda controller recreates a deleted StatefulSet if a racing + // pre-pause reconcile is still in flight; a recreated STS would re-adopt + // the orphaned pods and corrupt the broker-count assertions. Fail fast + // with a clear message instead (mirrors the integration-test guard). + require.Eventually(t, func() bool { + var recreated appsv1.StatefulSet + return apierrors.IsNotFound(t.Get(ctx, key, &recreated)) + }, time.Minute, 2*time.Second, "StatefulSet %q still exists after orphan-delete (recreated by a racing reconcile?)", clusterName) t.Logf("Orphan-deleted StatefulSet %q", clusterName) } @@ -76,7 +86,13 @@ func createBrokerCRsForCluster(ctx context.Context, t framework.TestingT, cluste for i, pod := range pods.Items { brokerName := fmt.Sprintf("%s-%d", key.Name, i) - specBytes, err := yaml.Marshal(pod.Spec) + // The live pod spec carries instance-specific scheduling state. + // NodeName in particular would pin every replacement pod to the + // original node — if that node is gone, scheduling times out. + podSpec := pod.Spec + podSpec.NodeName = "" + + specBytes, err := yaml.Marshal(podSpec) require.NoError(t, err) checksum := fmt.Sprintf("%x", sha256.Sum256(specBytes)) @@ -108,7 +124,7 @@ func createBrokerCRsForCluster(ctx context.Context, t framework.TestingT, cluste PodTemplate: redpandav1alpha2.BrokerPodTemplate{ Labels: pod.Labels, Annotations: map[string]string{redpandav1alpha2.BrokerConfigChecksumAnnotation: checksum}, - Spec: pod.Spec, + Spec: podSpec, }, Storage: redpandav1alpha2.BrokerStorage{ ExistingClaims: existingClaims, From ac2299b38047c1ee62103777e3640ea9c618a157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:26:07 +0200 Subject: [PATCH 18/29] fixup! operator: add the Broker CR reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review minors (reconciler side): - DeadNodePVCs no longer Retain-patches PVs backing ExistingClaims the caller then declines to remediate: exclusion happens up front, so an admin-set reclaimPolicy=Delete is not silently overridden (security). - reconcilePVCAdoption early-returns on decommissioning brokers instead of re-adopting claims the completion branch is about to delete. - clusters RBAC narrowed to get (list/watch unused); roles regenerated. - SkipNameValidation rationale comment; self-heal doc comments rephrased to future tense — the owning cluster controller is not wired up yet. --- .../files/rbac/v2-manager.ClusterRole.yaml | 2 - operator/config/rbac/itemized/v2-manager.yaml | 2 - .../controller/pvcunbinder/pvcunbinder.go | 15 ++++++- .../controller/redpanda/broker_controller.go | 43 +++++++++---------- .../controller/redpanda/testdata/role.yaml | 2 - 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/operator/chart/files/rbac/v2-manager.ClusterRole.yaml b/operator/chart/files/rbac/v2-manager.ClusterRole.yaml index de1fb5a793..e8ab370ed3 100644 --- a/operator/chart/files/rbac/v2-manager.ClusterRole.yaml +++ b/operator/chart/files/rbac/v2-manager.ClusterRole.yaml @@ -282,5 +282,3 @@ rules: - clusters verbs: - get - - list - - watch diff --git a/operator/config/rbac/itemized/v2-manager.yaml b/operator/config/rbac/itemized/v2-manager.yaml index 2c444041a8..665d546e48 100644 --- a/operator/config/rbac/itemized/v2-manager.yaml +++ b/operator/config/rbac/itemized/v2-manager.yaml @@ -282,5 +282,3 @@ rules: - clusters verbs: - get - - list - - watch diff --git a/operator/internal/controller/pvcunbinder/pvcunbinder.go b/operator/internal/controller/pvcunbinder/pvcunbinder.go index 029de568aa..43ca110f20 100644 --- a/operator/internal/controller/pvcunbinder/pvcunbinder.go +++ b/operator/internal/controller/pvcunbinder/pvcunbinder.go @@ -1178,14 +1178,27 @@ func NodeFromPVAffinity(pv *corev1.PersistentVolume) string { // longer exist. As a side effect, each affected PV's reclaim policy is // patched to Retain so the storage survives PVC deletion. // +// PVC names listed in exclude are skipped entirely — they are neither +// returned nor Retain-patched. Callers that will not remediate a claim +// (e.g. externally-managed ExistingClaims) must exclude it here: flipping +// the reclaim policy of a PV the caller then declines to touch would +// silently override an admin's `Delete` policy and strand Released volumes. +// // apiReader must be an uncached client for accurate Node existence checks. -func DeadNodePVCs(ctx context.Context, c client.Client, apiReader client.Reader, pod *corev1.Pod) ([]corev1.PersistentVolumeClaim, error) { +func DeadNodePVCs(ctx context.Context, c client.Client, apiReader client.Reader, pod *corev1.Pod, exclude ...string) ([]corev1.PersistentVolumeClaim, error) { l := log.FromContext(ctx) + excluded := make(map[string]bool, len(exclude)) + for _, name := range exclude { + excluded[name] = true + } var affected []corev1.PersistentVolumeClaim for i := range pod.Spec.Volumes { if pod.Spec.Volumes[i].PersistentVolumeClaim == nil { continue } + if excluded[pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName] { + continue + } var pvc corev1.PersistentVolumeClaim if err := c.Get(ctx, client.ObjectKey{ Name: pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName, diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index a9cb095988..b9d1043bf8 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -46,7 +46,7 @@ import ( // +kubebuilder:rbac:groups=cluster.redpanda.com,resources=brokers,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cluster.redpanda.com,resources=brokers/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cluster.redpanda.com,resources=brokers/finalizers,verbs=update -// +kubebuilder:rbac:groups=redpanda.vectorized.io,resources=clusters,verbs=get;list;watch +// +kubebuilder:rbac:groups=redpanda.vectorized.io,resources=clusters,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get;list;patch // +kubebuilder:rbac:groups="",resources=nodes,verbs=get @@ -72,6 +72,8 @@ type BrokerReconciler struct { func SetupBrokerController(_ context.Context, mgr multicluster.Manager, clientFactory internalclient.ClientFactory, namespace string, unbindPVCsAfter time.Duration) error { return mcbuilder.ControllerManagedBy(mgr).WithOptions(ctrlcontroller.TypedOptions[mcreconcile.Request]{ + // Tests register several reconcilers against one manager under the + // same default controller names; production registers each once. SkipNameValidation: ptr.To(true), }).For( &redpandav1alpha2.Broker{}, @@ -325,6 +327,11 @@ func (r *BrokerReconciler) reconcilePVCAdoption(ctx context.Context, state *brok if state.pod == nil { return ctrl.Result{}, nil } + // Don't re-adopt claims on a decommissioning broker: the completion + // branch is about to delete them anyway. + if state.broker.Spec.Decommission { + return ctrl.Result{}, nil + } l := log.FromContext(ctx) k8sClient := cluster.GetClient() scheme := cluster.GetScheme() @@ -809,9 +816,10 @@ func (r *BrokerReconciler) executeDecommission(ctx context.Context, clusterName // deletion alone — and a Stuck result (e.g. last-broker guard) blocks // deletion instead of falling through to pod removal. // - Raw deletion while the owning cluster is alive RELEASES the pod and -// PVCs (ownerRefs stripped): the broker keeps running, data survives, and -// the cluster controller recreates a Broker CR that re-adopts the pod — -// accidental deletion self-heals without a restart. +// PVCs (ownerRefs stripped): the broker keeps running and data survives. +// Once an owning cluster controller manages Broker CRs (none is wired up +// yet), it will recreate a Broker CR that re-adopts the pod — accidental +// deletion then self-heals without a restart. // - When the owning cluster itself is being deleted, the propagated // deletion policy decides: "cascade" (default) lets the GC delete pod and // PVCs with the CR — whole-cluster teardown, decommission is pointless — @@ -1073,26 +1081,17 @@ func (r *BrokerReconciler) remediatePVAffinity(ctx context.Context, l logr.Logge } } - affectedPVCs, err := pvcunbinder.DeadNodePVCs(ctx, k8sClient, apiReader, pod) - if err != nil { - return false, err - } - if len(affectedPVCs) == 0 { - return false, nil - } - - // Skip ExistingClaims — the controller doesn't own their spec. - existingClaimNames := map[string]bool{} + // ExistingClaims are excluded up front: the controller doesn't own + // their spec, so they are neither remediated NOR Retain-patched — + // flipping the reclaim policy of a PV we then decline to touch would + // override the admin's own policy. + var existingClaimNames []string for _, ec := range broker.Spec.Storage.ExistingClaims { - existingClaimNames[ec.Name] = true + existingClaimNames = append(existingClaimNames, ec.Name) } - var remediable []corev1.PersistentVolumeClaim - for _, pvc := range affectedPVCs { - if existingClaimNames[pvc.Name] { - l.Info("ExistingClaim PVC stuck on dead node, skipping (admin must handle)", "pvc", pvc.Name) - continue - } - remediable = append(remediable, pvc) + remediable, err := pvcunbinder.DeadNodePVCs(ctx, k8sClient, apiReader, pod, existingClaimNames...) + if err != nil { + return false, err } if len(remediable) == 0 { return false, nil diff --git a/operator/internal/controller/redpanda/testdata/role.yaml b/operator/internal/controller/redpanda/testdata/role.yaml index 88d88e09f6..e8e3ad24bb 100644 --- a/operator/internal/controller/redpanda/testdata/role.yaml +++ b/operator/internal/controller/redpanda/testdata/role.yaml @@ -309,8 +309,6 @@ rules: - clusters verbs: - get - - list - - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role From 039cceb031fd512b9484a5a5eec6c5cc41e5871e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:26:07 +0200 Subject: [PATCH 19/29] fixup! operator: add Broker CRD API, conditions, and roll-grant feature flags Review minors (API side): - BuildPod deep-copies the pod template: its maps and PodSpec shared memory with the (often informer-cached) Broker object, so a caller mutating the built pod would corrupt the CR. Latent today, but cheap to fix. - Direct unit coverage for the rotation contract: PodOutdated (both rotation keys, missing-annotation drift, unset-desired-key semantics) and BuildPod (isolation from the template). - BrokerDeletionPolicy propagation comment rephrased to future tense. --- .../api/redpanda/v1alpha2/broker_helpers.go | 10 ++- .../redpanda/v1alpha2/broker_helpers_test.go | 75 +++++++++++++++++++ operator/pkg/feature/flags.go | 5 +- 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/operator/api/redpanda/v1alpha2/broker_helpers.go b/operator/api/redpanda/v1alpha2/broker_helpers.go index 7f4b85e526..ec1a490b80 100644 --- a/operator/api/redpanda/v1alpha2/broker_helpers.go +++ b/operator/api/redpanda/v1alpha2/broker_helpers.go @@ -50,13 +50,17 @@ func (b *Broker) PodOutdated(pod *corev1.Pod) bool { } func (b *Broker) BuildPod(podName string) *corev1.Pod { + // Deep-copy the template: its PodSpec, labels and annotations otherwise + // share memory with the Broker object (often an informer-cache copy) — + // any caller mutating the returned pod would corrupt the CR. + tpl := b.Spec.PodTemplate.DeepCopy() pod := &corev1.Pod{ - Spec: b.Spec.PodTemplate.Spec, + Spec: tpl.Spec, ObjectMeta: metav1.ObjectMeta{ Name: podName, Namespace: b.Namespace, - Annotations: b.Spec.PodTemplate.Annotations, - Labels: b.Spec.PodTemplate.Labels, + Annotations: tpl.Annotations, + Labels: tpl.Labels, }, } return pod diff --git a/operator/api/redpanda/v1alpha2/broker_helpers_test.go b/operator/api/redpanda/v1alpha2/broker_helpers_test.go index 9838468fc0..506d007ee0 100644 --- a/operator/api/redpanda/v1alpha2/broker_helpers_test.go +++ b/operator/api/redpanda/v1alpha2/broker_helpers_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) @@ -61,3 +62,77 @@ func TestBrokerPodName(t *testing.T) { } assert.Equal(t, "rp-high-mem-1", nodePoolBroker.PodName()) } + +func TestPodOutdated(t *testing.T) { + broker := &Broker{ + Spec: BrokerSpec{ + PodTemplate: BrokerPodTemplate{ + Annotations: map[string]string{ + BrokerConfigChecksumAnnotation: "checksum-a", + BrokerClusterConfigVersionAnnotation: "7", + }, + }, + }, + } + pod := func(annotations map[string]string) *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: annotations}} + } + + // In-sync pod: both rotation keys match. + assert.False(t, broker.PodOutdated(pod(map[string]string{ + BrokerConfigChecksumAnnotation: "checksum-a", + BrokerClusterConfigVersionAnnotation: "7", + }))) + + // Checksum drift demands rotation. + assert.True(t, broker.PodOutdated(pod(map[string]string{ + BrokerConfigChecksumAnnotation: "checksum-b", + BrokerClusterConfigVersionAnnotation: "7", + }))) + + // Restart-marker drift demands rotation. + assert.True(t, broker.PodOutdated(pod(map[string]string{ + BrokerConfigChecksumAnnotation: "checksum-a", + BrokerClusterConfigVersionAnnotation: "6", + }))) + + // A missing pod annotation counts as drift when the template sets one. + assert.True(t, broker.PodOutdated(pod(map[string]string{ + BrokerConfigChecksumAnnotation: "checksum-a", + }))) + + // An UNSET desired key never demands rotation, whatever the pod has. + broker.Spec.PodTemplate.Annotations = map[string]string{} + assert.False(t, broker.PodOutdated(pod(map[string]string{ + BrokerConfigChecksumAnnotation: "anything", + }))) +} + +func TestBuildPodDeepCopies(t *testing.T) { + broker := &Broker{ + ObjectMeta: metav1.ObjectMeta{Name: "rp-0", Namespace: "ns"}, + Spec: BrokerSpec{ + ClusterRef: ClusterRef{Name: "rp"}, + NetworkIndex: ptr.To(int32(0)), + PodTemplate: BrokerPodTemplate{ + Labels: map[string]string{"a": "1"}, + Annotations: map[string]string{BrokerConfigChecksumAnnotation: "checksum-a"}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "redpanda", Image: "redpanda:v1"}}, + }, + }, + }, + } + + pod := broker.BuildPod(broker.PodName()) + assert.Equal(t, "rp-0", pod.Name) + assert.Equal(t, "checksum-a", pod.Annotations[BrokerConfigChecksumAnnotation]) + + // Mutating the built pod must not corrupt the Broker's template. + pod.Annotations[BrokerConfigChecksumAnnotation] = "mutated" + pod.Labels["a"] = "mutated" + pod.Spec.Containers[0].Image = "mutated" + assert.Equal(t, "checksum-a", broker.Spec.PodTemplate.Annotations[BrokerConfigChecksumAnnotation]) + assert.Equal(t, "1", broker.Spec.PodTemplate.Labels["a"]) + assert.Equal(t, "redpanda:v1", broker.Spec.PodTemplate.Spec.Containers[0].Image) +} diff --git a/operator/pkg/feature/flags.go b/operator/pkg/feature/flags.go index 22639fd127..e0614158b8 100644 --- a/operator/pkg/feature/flags.go +++ b/operator/pkg/feature/flags.go @@ -92,8 +92,9 @@ var ( // when its owning cluster is deleted: "cascade" (default) lets the GC // delete them with the Broker CR — whole-cluster teardown — while // "orphan" releases them (ownerRefs stripped, data survives). Users set - // it on the Cluster; the cluster controller propagates it onto each - // Broker CR so it stays readable after the Cluster object is gone. + // it on the Cluster; the owning cluster controller (once wired up) will + // propagate it onto each Broker CR so it stays readable after the + // Cluster object is gone. // Deletion of a single Broker CR while its cluster is alive always // releases the pod and PVCs, regardless of this policy. // Not registered in any bundle — never auto-set. From 1907e961de6011f7dcb5fc97a634d71c8b9e4436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:27:44 +0200 Subject: [PATCH 20/29] fixup! operator: add Broker controller integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review minors (integration tests): - TestFinalizerRawDeletionReleases now also asserts the PVCs survive with the CR ownerRefs stripped — the test claimed the release contract but never looked at a PVC, so a regression that stopped stripping PVC ownerRefs stayed green. - TestFinalizerPodNotOwned gives the pod a FOREIGN controller ownerRef: an ownerless pod was adopted by the reconciler before the deletion, so the test exercised the release path instead of the rollback branch it documents. - TestFinalizerPodGone makes pod creation impossible (invalid container name): pod-ensure is unconditional, so the pod-gone branch was raced rather than deterministically hit. --- .../redpanda/broker_controller_test.go | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/operator/internal/controller/redpanda/broker_controller_test.go b/operator/internal/controller/redpanda/broker_controller_test.go index 2fdab38505..79ddc35871 100644 --- a/operator/internal/controller/redpanda/broker_controller_test.go +++ b/operator/internal/controller/redpanda/broker_controller_test.go @@ -31,6 +31,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -197,6 +198,10 @@ func (s *BrokerControllerSuite) TestFinalizerPodGone() { defer cancel() broker := s.minimalBroker("no-such-cluster") + // Make pod creation impossible (invalid container name): pod-ensure is + // unconditional, so an ordinary Broker would have a pod by the time the + // deletion reconcile runs and the pod-gone branch would never be hit. + broker.Spec.PodTemplate.Spec.Containers[0].Name = "Invalid_Container_Name" require.NoError(t, c.Create(ctx, broker)) s.waitForFinalizer(t, ctx, c, broker) @@ -204,6 +209,12 @@ func (s *BrokerControllerSuite) TestFinalizerPodGone() { require.NoError(t, c.Delete(ctx, broker)) s.waitForDeletion(t, ctx, c, broker) + + var pods corev1.PodList + require.NoError(t, c.List(ctx, &pods, client.InNamespace(broker.Namespace))) + for _, p := range pods.Items { + assert.NotEqual(t, broker.PodName(), p.Name, "no pod should ever have existed for the deleted Broker") + } } // TestFinalizerPodNotOwned verifies the rollback case: if the pod exists but is @@ -217,9 +228,19 @@ func (s *BrokerControllerSuite) TestFinalizerPodNotOwned() { clusterName := "rollback-" + testenv.RandString(4) podName := clusterName + "-0" + // The pod carries a FOREIGN controller ownerRef: an ownerless pod would + // be adopted by the reconciler before the deletion below, silently + // turning this into a release-path test instead of the rollback branch. pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: clusterName, + UID: types.UID("foreign-owner-uid"), + Controller: ptr.To(true), + }}, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{{ @@ -375,6 +396,19 @@ func (s *BrokerControllerSuite) TestFinalizerRawDeletionReleases() { require.NoError(t, c.Get(ctx, client.ObjectKey{Name: targetPodName, Namespace: rp.Namespace}, &pod)) assert.Nil(t, metav1.GetControllerOf(&pod), "pod should have been released from the deleted Broker CR") + // The PVCs survive too, with the CR's ownerRefs stripped — this is the + // data-preservation half of the release contract. + for _, vol := range pod.Spec.Volumes { + if vol.PersistentVolumeClaim == nil { + continue + } + var pvc corev1.PersistentVolumeClaim + require.NoError(t, c.Get(ctx, client.ObjectKey{Name: vol.PersistentVolumeClaim.ClaimName, Namespace: rp.Namespace}, &pvc)) + for _, ref := range pvc.OwnerReferences { + assert.NotEqual(t, target.UID, ref.UID, "PVC %s still owned by the deleted Broker CR", pvc.Name) + } + } + // Membership is untouched: still 3 brokers. bs, err := admin.Brokers(ctx) require.NoError(t, err) From 547a028d5520d07604bcdd199f63803d97b4f9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 15:28:55 +0200 Subject: [PATCH 21/29] fixup! operator: add Broker CR acceptance coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review minors (acceptance): - The adoption scenario now snapshots pod UIDs before the migration and asserts they are unchanged after — "adopted without a restart" was claimed but a delete-and-recreate regression would still have shown 3 Running brokers and passed. - clusterAdminAPIShouldShowBrokers closes the admin client it builds on every poll iteration (previously leaked ~60 clients over a full wait). - allBrokerCRsRunning and the createBrokerCRsForCluster cleanup are scoped to the cluster-name label instead of the whole namespace. --- .../features/broker-crd-adoption.feature | 3 ++ acceptance/steps/broker.go | 47 ++++++++++++++++++- acceptance/steps/register.go | 2 + 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/acceptance/features/broker-crd-adoption.feature b/acceptance/features/broker-crd-adoption.feature index b8533bf0d0..b209b0ad74 100644 --- a/acceptance/features/broker-crd-adoption.feature +++ b/acceptance/features/broker-crd-adoption.feature @@ -5,10 +5,13 @@ Feature: Broker CRD pod adoption Given I create a basic cluster "broker-test" with 3 nodes And cluster "broker-test" is stable with 3 nodes When I pause reconciliation on cluster "broker-test" + And I snapshot pod UIDs for cluster "broker-test" And I orphan-delete the StatefulSet for cluster "broker-test" And I create Broker CRs for cluster "broker-test" # Adoption and pod-ensure are not gated on roll-grants (RFC Q5). Then all Broker CRs for cluster "broker-test" should be Running + # Adoption must not restart anything: same pods, same UIDs. + And pods for cluster "broker-test" should have the same UIDs as the snapshot And cluster "broker-test" admin API should show 3 brokers When I update Broker "broker-test-0" pod template with env "ROTATION_TEST=applied" for cluster "broker-test" # Rotation IS gated: an outdated pod without a grant must stay put... diff --git a/acceptance/steps/broker.go b/acceptance/steps/broker.go index deb131a3b9..d205e3df8b 100644 --- a/acceptance/steps/broker.go +++ b/acceptance/steps/broker.go @@ -138,13 +138,53 @@ func createBrokerCRsForCluster(ctx context.Context, t framework.TestingT, cluste t.Cleanup(func(ctx context.Context) { t := framework.T(ctx) var brokers redpandav1alpha2.BrokerList - _ = t.List(ctx, &brokers, runtimeclient.InNamespace(key.Namespace)) + _ = t.List(ctx, &brokers, runtimeclient.InNamespace(key.Namespace), runtimeclient.MatchingLabels{ + redpandav1alpha2.ClusterNameLabel: clusterName, + }) for i := range brokers.Items { _ = t.Delete(ctx, &brokers.Items[i]) } }) } +type podUIDSnapshotKey struct{ cluster string } + +// snapshotPodUIDs records the UID of every redpanda pod of the cluster so a +// later step can assert none of them was deleted/recreated. +func snapshotPodUIDs(ctx context.Context, t framework.TestingT, clusterName string) context.Context { + key := t.ResourceKey(clusterName) + var pods corev1.PodList + require.NoError(t, t.List(ctx, &pods, runtimeclient.InNamespace(key.Namespace), runtimeclient.MatchingLabels{ + "app.kubernetes.io/instance": clusterName, + "app.kubernetes.io/name": "redpanda", + })) + require.NotEmpty(t, pods.Items, "no pods to snapshot for cluster %q", clusterName) + uids := map[string]string{} + for _, p := range pods.Items { + uids[p.Name] = string(p.UID) + } + t.Logf("Snapshot %d pod UIDs for cluster %q: %v", len(uids), clusterName, uids) + return context.WithValue(ctx, podUIDSnapshotKey{clusterName}, uids) +} + +func podUIDsShouldBeUnchanged(ctx context.Context, t framework.TestingT, clusterName string) { + snap, ok := ctx.Value(podUIDSnapshotKey{clusterName}).(map[string]string) + require.True(t, ok, "no pod UID snapshot found for cluster %q", clusterName) + + key := t.ResourceKey(clusterName) + var pods corev1.PodList + require.NoError(t, t.List(ctx, &pods, runtimeclient.InNamespace(key.Namespace), runtimeclient.MatchingLabels{ + "app.kubernetes.io/instance": clusterName, + "app.kubernetes.io/name": "redpanda", + })) + current := map[string]string{} + for _, p := range pods.Items { + current[p.Name] = string(p.UID) + } + require.Equal(t, snap, current, "pod UIDs changed for cluster %q — pods were restarted", clusterName) + t.Logf("All %d pod UIDs for cluster %q are unchanged", len(current), clusterName) +} + func grantRollGrantToBroker(ctx context.Context, t framework.TestingT, brokerName string) { key := t.ResourceKey(brokerName) var broker redpandav1alpha2.Broker @@ -166,7 +206,9 @@ func allBrokerCRsRunning(ctx context.Context, t framework.TestingT, clusterName key := t.ResourceKey(clusterName) require.Eventually(t, func() bool { var brokers redpandav1alpha2.BrokerList - if err := t.List(ctx, &brokers, runtimeclient.InNamespace(key.Namespace)); err != nil { + if err := t.List(ctx, &brokers, runtimeclient.InNamespace(key.Namespace), runtimeclient.MatchingLabels{ + redpandav1alpha2.ClusterNameLabel: clusterName, + }); err != nil { return false } if len(brokers.Items) == 0 { @@ -283,6 +325,7 @@ func clusterAdminAPIShouldShowBrokers(ctx context.Context, t framework.TestingT, t.Logf("Failed to create admin client: %v", err) return false } + defer admin.Close() brokers, err := admin.Brokers(ctx) if err != nil { t.Logf("Failed to get brokers: %v", err) diff --git a/acceptance/steps/register.go b/acceptance/steps/register.go index 5726d561b4..f80ba47205 100644 --- a/acceptance/steps/register.go +++ b/acceptance/steps/register.go @@ -180,6 +180,8 @@ func init() { framework.RegisterStep(`^I pause reconciliation on cluster "([^"]*)"$`, pauseReconciliation) framework.RegisterStep(`^I orphan-delete the StatefulSet for cluster "([^"]*)"$`, orphanDeleteStatefulSet) framework.RegisterStep(`^I create Broker CRs for cluster "([^"]*)"$`, createBrokerCRsForCluster) + framework.RegisterStep(`^I snapshot pod UIDs for cluster "([^"]*)"$`, snapshotPodUIDs) + framework.RegisterStep(`^pods for cluster "([^"]*)" should have the same UIDs as the snapshot$`, podUIDsShouldBeUnchanged) framework.RegisterStep(`^I grant a roll-grant to Broker "([^"]*)"$`, grantRollGrantToBroker) framework.RegisterStep(`^all Broker CRs for cluster "([^"]*)" should be Running$`, allBrokerCRsRunning) framework.RegisterStep(`^I set decommission on Broker "([^"]*)"$`, setDecommissionOnBroker) From 39b0788cb8bfee2b908db0007f0ef4abcf8f5b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 17:30:25 +0200 Subject: [PATCH 22/29] fixup! operator: add the Broker CR reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M6 refinement, found by the integration suite: plain ambiguity-refusal in resolveBroker wedges PV-remediation recovery forever — the dead predecessor entry lingers in the membership list under the SAME reused pod name until ghost decommissioning, so the resolver never returned a match and identity adoption never happened. At most one live process can advertise the address at a time, so a unique active+alive match among ambiguous candidates is authoritative. A lone dead entry remains un-adoptable, preserving the original aliasing fix. --- .../controller/redpanda/broker_controller.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index b9d1043bf8..e7b544954b 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -1162,8 +1162,22 @@ func (r *BrokerReconciler) resolveBroker(ctx context.Context, clusterName string case 1: return &matches[0], nil default: + // A dead predecessor's entry (PV remediation, broker replacement) + // lingers under the SAME reused pod name until it is ghost- + // decommissioned, so plain ambiguity-refusal would wedge recovery + // forever. At most one LIVE process can advertise the address at a + // time — a unique active+alive match is authoritative. + var live []rpadmin.Broker + for _, b := range matches { + if brokerActiveAndAlive(&b) { + live = append(live, b) + } + } + if len(live) == 1 { + return &live[0], nil + } log.FromContext(ctx).Info("ambiguous cluster-membership match for pod, refusing to guess", - "pod", podName, "matches", len(matches)) + "pod", podName, "matches", len(matches), "alive", len(live)) return nil, nil } } From 7b22ebf165823cb7809e40814d7edb20d845b6e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 17:30:25 +0200 Subject: [PATCH 23/29] fixup! operator: add Broker controller integration tests TestFinalizerPodNotOwned gave the pod a fabricated foreign ownerRef UID; the garbage collector treats a reference to a non-existent owner as an orphan and deleted the pod out from under the test. The foreign owner is now a real object (ConfigMap). --- .../controller/redpanda/broker_controller_test.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller_test.go b/operator/internal/controller/redpanda/broker_controller_test.go index 79ddc35871..972a54531d 100644 --- a/operator/internal/controller/redpanda/broker_controller_test.go +++ b/operator/internal/controller/redpanda/broker_controller_test.go @@ -31,7 +31,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -231,14 +230,20 @@ func (s *BrokerControllerSuite) TestFinalizerPodNotOwned() { // The pod carries a FOREIGN controller ownerRef: an ownerless pod would // be adopted by the reconciler before the deletion below, silently // turning this into a release-path test instead of the rollback branch. + // The owner must be a REAL object — the GC deletes orphans of + // non-existent owners, which would race the whole test. + foreignOwner := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: clusterName + "-owner"}, + } + require.NoError(t, c.Create(ctx, foreignOwner)) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: clusterName, - UID: types.UID("foreign-owner-uid"), + APIVersion: "v1", + Kind: "ConfigMap", + Name: foreignOwner.Name, + UID: foreignOwner.UID, Controller: ptr.To(true), }}, }, From 805cc12a2479b7207cbd8757aaf0a087a033f39c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 18:09:53 +0200 Subject: [PATCH 24/29] fixup! operator: add the Broker CR reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up (Paweł): resolveBroker returned (nil, nil) for the no-match case — an antipattern with a (value, error) signature, and nil is the COMMON case while a broker is joining, so a future call site that only checks err would panic in steady state rather than in tests. The signature is now (resolved, found, err) so every caller branches on found explicitly; ambiguous matches report found=false, same contract as no match. --- .../controller/redpanda/broker_controller.go | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index e7b544954b..9679dd899d 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -462,12 +462,12 @@ func (r *BrokerReconciler) reconcileBrokerRegistration(ctx context.Context, stat l := log.FromContext(ctx) podName := broker.PodName() - resolved, err := r.resolveBroker(ctx, state.clusterName, broker, state.pod, podName) + resolved, found, err := r.resolveBroker(ctx, state.clusterName, broker, state.pod, podName) if err != nil { l.Info("could not resolve broker ID, will retry", "error", err) return ctrl.Result{RequeueAfter: requeueShort}, nil } - if resolved == nil { + if !found { if broker.Status.BrokerID != nil { // Was registered, no longer a member (e.g. removed out of // band): report unverified but let the chain continue. @@ -535,8 +535,8 @@ func (r *BrokerReconciler) reconcileDecommission(ctx context.Context, state *bro // or an intent set before the first registration (or right after a // status-update race) never actually starts the decommission and the // broker stays a full cluster member while reporting Decommissioning. - resolved, err := r.resolveBroker(ctx, state.clusterName, broker, state.pod, broker.PodName()) - if err != nil || resolved == nil || resolved.MembershipStatus != rpadmin.MembershipStatusActive { + resolved, found, err := r.resolveBroker(ctx, state.clusterName, broker, state.pod, broker.PodName()) + if err != nil || !found || resolved.MembershipStatus != rpadmin.MembershipStatusActive { if err != nil { log.FromContext(ctx).Info("could not resolve broker ID before decommission, will retry", "error", err) } @@ -861,12 +861,12 @@ func (r *BrokerReconciler) reconcileDelete(ctx context.Context, l logr.Logger, k // it live rather than skipping the decommission and leaving a dead // membership entry behind. if broker.Status.BrokerID == nil { - resolved, err := r.resolveBroker(ctx, clusterName, broker, &pod, podName) + resolved, found, err := r.resolveBroker(ctx, clusterName, broker, &pod, podName) if err != nil { l.Info("could not resolve broker ID before decommission, will retry", "error", err) return ctrl.Result{RequeueAfter: requeueShort}, nil } - if resolved != nil && resolved.MembershipStatus == rpadmin.MembershipStatusActive { + if found && resolved.MembershipStatus == rpadmin.MembershipStatusActive { broker.Status.BrokerID = ptr.To(int32(resolved.NodeID)) } } @@ -1122,24 +1122,28 @@ func (r *BrokerReconciler) disableMaintenanceMode(ctx context.Context, clusterNa } // resolveBroker finds the cluster-membership entry backing this Broker's -// pod. Matching handles every advertised-address form seen in the wild: -// per-pod FQDN (matched by first DNS label), "host:port", bare host, and — -// when the pod is known — a bare pod IP. More than one distinct match is -// ambiguous and reported as no match rather than guessed at: right after a +// pod. found reports whether an authoritative entry exists — false is the +// COMMON case while a broker is still joining, so callers must branch on it +// explicitly rather than relying on error or nilness. +// +// Matching handles every advertised-address form seen in the wild: per-pod +// FQDN (matched by first DNS label), "host:port", bare host, and — when the +// pod is known — a bare pod IP. More than one distinct match is ambiguous +// and reported as not found rather than guessed at: right after a // decommission the membership list can briefly retain the dead // predecessor's entry under the SAME pod name (a replacement reuses it). // Callers that ADOPT the resolved id must additionally check the entry is // an active, alive member — see reconcileBrokerRegistration. -func (r *BrokerReconciler) resolveBroker(ctx context.Context, clusterName string, broker *redpandav1alpha2.Broker, pod *corev1.Pod, podName string) (*rpadmin.Broker, error) { +func (r *BrokerReconciler) resolveBroker(ctx context.Context, clusterName string, broker *redpandav1alpha2.Broker, pod *corev1.Pod, podName string) (resolved *rpadmin.Broker, found bool, err error) { admin, err := r.ClientFactory.RedpandaAdminClientForCluster(ctx, broker, clusterName) if err != nil { - return nil, err + return nil, false, err } defer admin.Close() brokers, err := admin.Brokers(ctx) if err != nil { - return nil, err + return nil, false, err } var matches []rpadmin.Broker @@ -1158,9 +1162,9 @@ func (r *BrokerReconciler) resolveBroker(ctx context.Context, clusterName string } switch len(matches) { case 0: - return nil, nil + return nil, false, nil case 1: - return &matches[0], nil + return &matches[0], true, nil default: // A dead predecessor's entry (PV remediation, broker replacement) // lingers under the SAME reused pod name until it is ghost- @@ -1174,11 +1178,11 @@ func (r *BrokerReconciler) resolveBroker(ctx context.Context, clusterName string } } if len(live) == 1 { - return &live[0], nil + return &live[0], true, nil } log.FromContext(ctx).Info("ambiguous cluster-membership match for pod, refusing to guess", "pod", podName, "matches", len(matches), "alive", len(live)) - return nil, nil + return nil, false, nil } } From 46f4ad16cd799dd3d357a28d817e40338d4f6450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 13 Jul 2026 18:18:02 +0200 Subject: [PATCH 25/29] fixup! operator: add the Broker CR reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up (Paweł, amends M8): dead-node PV remediation no longer stays disabled when --unbind-pvcs-after is left at its zero default — SetupBrokerController honors the flag when set and otherwise falls back to a 5m default of its own. Remediation only fires for pods stuck Pending on PV node affinity whose Node object is gone from the API, and is roll-grant gated on top, so default-on is safe; a TODO notes the dedicated --broker-unbind-pvcs-after flag for the GA config surface. Flag help and changelog updated accordingly. --- .../operator-Added-20260709-120000.yaml | 2 +- operator/cmd/run/run.go | 2 +- .../controller/redpanda/broker_controller.go | 22 ++++++++++++++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.changes/unreleased/operator-Added-20260709-120000.yaml b/.changes/unreleased/operator-Added-20260709-120000.yaml index 86a807d5ff..fa00172c64 100644 --- a/.changes/unreleased/operator-Added-20260709-120000.yaml +++ b/.changes/unreleased/operator-Added-20260709-120000.yaml @@ -1,4 +1,4 @@ project: operator kind: Added -body: Broker custom resource and reconciler (experimental, behind --enable-broker) — per-broker pod/PVC lifecycle with grant-gated serialized rotation, leadership drain, node_id continuity verification, explicit-intent decommission with recommission and last-broker guard, dead-node PV remediation (requires --unbind-pvcs-after to be set), and full status conditions +body: Broker custom resource and reconciler (experimental, behind --enable-broker) — per-broker pod/PVC lifecycle with grant-gated serialized rotation, leadership drain, node_id continuity verification, explicit-intent decommission with recommission and last-broker guard, dead-node PV remediation (honors --unbind-pvcs-after when set, 5m default otherwise), and full status conditions time: 2026-07-09T12:00:00.000000000+02:00 diff --git a/operator/cmd/run/run.go b/operator/cmd/run/run.go index 2b9c986292..96e33f54fa 100644 --- a/operator/cmd/run/run.go +++ b/operator/cmd/run/run.go @@ -201,7 +201,7 @@ func (o *RunOptions) BindFlags(cmd *cobra.Command) { cmd.Flags().BoolVar(&o.ghostbuster, "unsafe-decommission-failed-brokers", false, "Set to enable decommissioning a failed broker that is configured but does not exist in the StatefulSet (ghost broker). This may result in invalidating valid data") _ = cmd.Flags().MarkHidden("unsafe-decommission-failed-brokers") cmd.Flags().StringSliceVar(&o.additionalControllers, "additional-controllers", []string{""}, fmt.Sprintf("which controllers to run, available: all, %s", strings.Join(availableControllers, ", "))) - cmd.Flags().DurationVar(&o.unbindPVCsAfter, "unbind-pvcs-after", 0, "if not zero, runs the PVCUnbinder controller which attempts to 'unbind' the PVCs' of Pods that are Pending for longer than the given duration; also a prerequisite for the Broker controller's dead-node PV remediation (--enable-broker), which stays disabled while this is zero") + cmd.Flags().DurationVar(&o.unbindPVCsAfter, "unbind-pvcs-after", 0, "if not zero, runs the PVCUnbinder controller which attempts to 'unbind' the PVCs' of Pods that are Pending for longer than the given duration; the Broker controller's dead-node PV remediation (--enable-broker) also uses this timeout when set, and falls back to its own default (5m) when zero") cmd.Flags().BoolVar(&o.allowPVRebinding, "allow-pv-rebinding", false, "DEPRECATED. When the PVCUnbinder fires, also clear the freed PV's ClaimRef so the disk can be reused if the node returns. Risks cross-broker disk swap when multiple PVs are cleared concurrently. Leave at false (the default) — the unbinder's pause-annotation, multi-pod auto-detect, and per-cluster serialization gates make this flag unnecessary in practice and unsafe in the cases where it would have been useful.") _ = cmd.Flags().MarkDeprecated("allow-pv-rebinding", "the gating checks added to the PVCUnbinder (pause annotation, multi-pod auto-detect, per-cluster serialization) supersede this flag; see the PVCUnbinder controller godoc") cmd.Flags().Var(&o.unbinderSelector, "unbinder-label-selector", "if provided, a Kubernetes label selector that will filter Pods to be considered by the PVCUnbinder.") diff --git a/operator/internal/controller/redpanda/broker_controller.go b/operator/internal/controller/redpanda/broker_controller.go index 9679dd899d..6263a086cf 100644 --- a/operator/internal/controller/redpanda/broker_controller.go +++ b/operator/internal/controller/redpanda/broker_controller.go @@ -53,6 +53,17 @@ import ( const brokerFinalizerName = "cluster.redpanda.com/broker-decommission" +// defaultUnbindPVCsAfter is the Broker controller's own dead-node PV +// remediation timeout, used when --unbind-pvcs-after is not set. Remediation +// must not silently stay disabled just because the (PVCUnbinder-oriented) +// flag was left at its zero default: it only fires for pods stuck Pending on +// PV node affinity whose Node OBJECT has been deleted from the API — a +// strong dead-node signal — and it is additionally roll-grant-gated, so the +// cluster controller serializes and health-gates it. +// TODO: give the Broker controller its own flag (e.g. +// --broker-unbind-pvcs-after) when the feature grows its GA config surface. +const defaultUnbindPVCsAfter = 5 * time.Minute + const requeueShort = 2 * time.Second // requeueDrain paces the leadership-drain poll during a granted rotation. @@ -71,6 +82,11 @@ type BrokerReconciler struct { } func SetupBrokerController(_ context.Context, mgr multicluster.Manager, clientFactory internalclient.ClientFactory, namespace string, unbindPVCsAfter time.Duration) error { + // --unbind-pvcs-after is honored when set; otherwise fall back to the + // controller's own default instead of disabling remediation. + if unbindPVCsAfter <= 0 { + unbindPVCsAfter = defaultUnbindPVCsAfter + } return mcbuilder.ControllerManagedBy(mgr).WithOptions(ctrlcontroller.TypedOptions[mcreconcile.Request]{ // Tests register several reconcilers against one manager under the // same default controller names; production registers each once. @@ -1063,10 +1079,10 @@ func (r *BrokerReconciler) ensureDrained(ctx context.Context, clusterName string // Returns true if remediation was performed (caller should requeue). // Returns false if the timeout hasn't elapsed or the node still exists. func (r *BrokerReconciler) remediatePVAffinity(ctx context.Context, l logr.Logger, k8sClient client.Client, apiReader client.Reader, broker *redpandav1alpha2.Broker, pod *corev1.Pod) (bool, error) { + // Unreachable through SetupBrokerController (which defaults the value); + // guards direct construction only. if r.UnbindPVCsAfter <= 0 { - l.Info("broker pod is stuck on PV node affinity but remediation is disabled; " + - "set --unbind-pvcs-after to a non-zero duration to enable dead-node PV remediation " + - "(the broker will otherwise stay Stuck until the node returns or the PVC is removed manually)") + l.Info("PV affinity remediation disabled (UnbindPVCsAfter=0); broker will stay Stuck until the node returns or the PVC is removed manually") return false, nil } From d030277d362fd3544d92f3216d2df4d2b2bfc4b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Tue, 14 Jul 2026 09:52:33 +0200 Subject: [PATCH 26/29] chore: regenerate golden files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Bojanowski --- .../testdata/template-cases.golden.txtar | 248 ------------------ 1 file changed, 248 deletions(-) diff --git a/operator/chart/testdata/template-cases.golden.txtar b/operator/chart/testdata/template-cases.golden.txtar index 6e1a50ce7b..98ae7e437e 100644 --- a/operator/chart/testdata/template-cases.golden.txtar +++ b/operator/chart/testdata/template-cases.golden.txtar @@ -600,8 +600,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1515,8 +1513,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -2209,8 +2205,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -3151,8 +3145,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -3865,8 +3857,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -4793,8 +4783,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -5651,8 +5639,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -6782,8 +6768,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -7581,8 +7565,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -8510,8 +8492,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -9208,8 +9188,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -10175,8 +10153,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -10961,8 +10937,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -11974,8 +11948,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -12793,8 +12765,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -13739,8 +13709,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -14440,8 +14408,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -15374,8 +15340,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -16077,8 +16041,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -17043,8 +17005,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -17744,8 +17704,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -18518,8 +18476,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -19233,8 +19189,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -20174,8 +20128,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -21051,8 +21003,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -22199,8 +22149,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -22909,8 +22857,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -23729,8 +23675,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -24418,8 +24362,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -25361,8 +25303,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -26070,8 +26010,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -27005,8 +26943,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -28081,8 +28017,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -29233,8 +29167,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -29943,8 +29875,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -30917,8 +30847,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -31776,8 +31704,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -32867,8 +32793,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -33947,8 +33871,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -35037,8 +34959,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -35898,8 +35818,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -37228,8 +37146,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -37937,8 +37853,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -38884,8 +38798,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -39760,8 +39672,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -41016,8 +40926,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -42078,8 +41986,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -43216,8 +43122,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -44345,8 +44249,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -45507,8 +45409,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -46227,8 +46127,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -47375,8 +47273,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -48244,8 +48140,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -49421,8 +49315,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -50820,8 +50712,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -52152,8 +52042,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -52972,8 +52860,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -53941,8 +53827,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -54804,8 +54688,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -56205,8 +56087,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -58645,8 +58525,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -60150,8 +60028,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -61823,8 +61699,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -63064,8 +62938,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -65471,8 +65343,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -66950,8 +66820,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -67769,8 +67637,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -69285,8 +69151,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -70776,8 +70640,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -72011,8 +71873,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -73586,8 +73446,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -75058,8 +74916,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -75898,8 +75754,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -77184,8 +77038,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -78192,8 +78044,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -79718,8 +79568,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -80537,8 +80385,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -81966,8 +81812,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -82788,8 +82632,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -83704,8 +83546,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -84398,8 +84238,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -85356,8 +85194,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -86141,8 +85977,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -87099,8 +86933,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -87883,8 +87715,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -88797,8 +88627,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -89491,8 +89319,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -90422,8 +90248,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -91116,8 +90940,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -92030,8 +91852,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -92724,8 +92544,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -93638,8 +93456,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -94332,8 +94148,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -95246,8 +95060,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -95940,8 +95752,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -96862,8 +96672,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -97556,8 +97364,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -98478,8 +98284,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -99172,8 +98976,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -100120,8 +99922,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -100814,8 +100614,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -101852,8 +101650,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -102546,8 +102342,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -103496,8 +103290,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -104190,8 +103982,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -105140,8 +104930,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -105834,8 +105622,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -106783,8 +106569,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -107493,8 +107277,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -108512,8 +108294,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -109313,8 +109093,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -110387,8 +110165,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -111188,8 +110964,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -112256,8 +112030,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -113057,8 +112829,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -114058,8 +113828,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -114859,8 +114627,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -115852,8 +115618,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -116637,8 +116401,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -117682,8 +117444,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -118376,8 +118136,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -119297,8 +119055,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -119991,8 +119747,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -120906,8 +120660,6 @@ rules: - clusters verbs: - get - - list - - watch --- # Source: operator/templates/entry-point.yaml apiVersion: rbac.authorization.k8s.io/v1 From 31d29a54b040de0a8b520b0869eab057016ff921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Tue, 14 Jul 2026 11:13:34 +0200 Subject: [PATCH 27/29] fixup! operator: add the Broker CR reconciler Simplification (ponytail review): DeadNodePVCs uses slices.Contains on the exclude list instead of building a map for 0-2 entries. --- operator/internal/controller/pvcunbinder/pvcunbinder.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/operator/internal/controller/pvcunbinder/pvcunbinder.go b/operator/internal/controller/pvcunbinder/pvcunbinder.go index 43ca110f20..f1176a7202 100644 --- a/operator/internal/controller/pvcunbinder/pvcunbinder.go +++ b/operator/internal/controller/pvcunbinder/pvcunbinder.go @@ -1187,16 +1187,12 @@ func NodeFromPVAffinity(pv *corev1.PersistentVolume) string { // apiReader must be an uncached client for accurate Node existence checks. func DeadNodePVCs(ctx context.Context, c client.Client, apiReader client.Reader, pod *corev1.Pod, exclude ...string) ([]corev1.PersistentVolumeClaim, error) { l := log.FromContext(ctx) - excluded := make(map[string]bool, len(exclude)) - for _, name := range exclude { - excluded[name] = true - } var affected []corev1.PersistentVolumeClaim for i := range pod.Spec.Volumes { if pod.Spec.Volumes[i].PersistentVolumeClaim == nil { continue } - if excluded[pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName] { + if slices.Contains(exclude, pod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName) { continue } var pvc corev1.PersistentVolumeClaim From ba1f657569efd1a4efef6b7274992d339ed4bf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Tue, 14 Jul 2026 11:13:34 +0200 Subject: [PATCH 28/29] fixup! operator: add Broker controller integration tests Simplification pass (ponytail review): grant encoding via feature.FormatRollGrant in all three test sites (no format drift vs prod), direct Get+IsNotFound for the pod-absence assertion, and suite-level importImages reused for the replacement-node re-import. --- .../redpanda/broker_controller_test.go | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/operator/internal/controller/redpanda/broker_controller_test.go b/operator/internal/controller/redpanda/broker_controller_test.go index 972a54531d..e1aa8436b1 100644 --- a/operator/internal/controller/redpanda/broker_controller_test.go +++ b/operator/internal/controller/redpanda/broker_controller_test.go @@ -14,7 +14,6 @@ import ( "crypto/sha256" "fmt" "os" - "strconv" "strings" "testing" "time" @@ -45,6 +44,7 @@ import ( "github.com/redpanda-data/redpanda-operator/operator/internal/statuses" "github.com/redpanda-data/redpanda-operator/operator/internal/testenv" internalclient "github.com/redpanda-data/redpanda-operator/operator/pkg/client" + "github.com/redpanda-data/redpanda-operator/operator/pkg/feature" "github.com/redpanda-data/redpanda-operator/pkg/multicluster" "github.com/redpanda-data/redpanda-operator/pkg/testutil" ) @@ -59,6 +59,9 @@ type BrokerControllerSuite struct { env *testenv.Env clientFactory internalclient.ClientFactory + // importImages is the image set newEnv loads into every cluster it + // builds; tests that add k3d nodes mid-run re-import it onto them. + importImages []string } var _ suite.SetupAllSuite = (*BrokerControllerSuite)(nil) @@ -104,6 +107,7 @@ func (s *BrokerControllerSuite) newEnv(t *testing.T, clusterName string) (*teste } } + s.importImages = importImages env := testenv.New(t, testenv.Options{ Name: clusterName, Scheme: controller.V2Scheme, @@ -209,11 +213,9 @@ func (s *BrokerControllerSuite) TestFinalizerPodGone() { s.waitForDeletion(t, ctx, c, broker) - var pods corev1.PodList - require.NoError(t, c.List(ctx, &pods, client.InNamespace(broker.Namespace))) - for _, p := range pods.Items { - assert.NotEqual(t, broker.PodName(), p.Name, "no pod should ever have existed for the deleted Broker") - } + var pod corev1.Pod + err := c.Get(ctx, client.ObjectKey{Name: broker.PodName(), Namespace: broker.Namespace}, &pod) + assert.True(t, apierrors.IsNotFound(err), "no pod should ever have existed for the deleted Broker, got err=%v", err) } // TestFinalizerPodNotOwned verifies the rollback case: if the pod exists but is @@ -557,7 +559,7 @@ func (s *BrokerControllerSuite) TestPodRotationWithoutGrant() { if target.Annotations == nil { target.Annotations = map[string]string{} } - target.Annotations["operator.redpanda.com/roll-grant"] = newChecksum + "/" + strconv.FormatInt(time.Now().Add(10*time.Minute).Unix(), 10) + target.Annotations["operator.redpanda.com/roll-grant"] = feature.FormatRollGrant(newChecksum, time.Now().Add(10*time.Minute)) require.NoError(t, c.Patch(ctx, target, p)) require.EventuallyWithT(t, func(ct *assert.CollectT) { @@ -689,7 +691,7 @@ func (s *BrokerControllerSuite) TestPVAffinityRemediation() { target.Annotations = map[string]string{} } checksum := target.Spec.PodTemplate.Annotations["config.redpanda.com/checksum"] - target.Annotations["operator.redpanda.com/roll-grant"] = checksum + "/" + strconv.FormatInt(time.Now().Add(30*time.Minute).Unix(), 10) + target.Annotations["operator.redpanda.com/roll-grant"] = feature.FormatRollGrant(checksum, time.Now().Add(30*time.Minute)) require.NoError(t, c.Patch(ctx, target, p)) // Create a replacement k3d node so the pod has somewhere to schedule, @@ -699,12 +701,7 @@ func (s *BrokerControllerSuite) TestPVAffinityRemediation() { // wait below would time out. t.Log("creating replacement k3d node") require.NoError(t, env.Host().CreateNode()) - require.NoError(t, env.Host().ImportImage("localhost/redpanda-operator:dev")) - if repo := os.Getenv("TEST_REDPANDA_REPO"); repo != "" { - if version := os.Getenv("TEST_REDPANDA_VERSION"); version != "" { - require.NoError(t, env.Host().ImportImage(fmt.Sprintf("%s:%s", repo, version))) - } - } + require.NoError(t, env.Host().ImportImage(s.importImages...)) // Wait for the broker to recover to Running. t.Log("waiting for Broker to recover to Running") @@ -842,7 +839,7 @@ func (s *BrokerControllerSuite) setupBrokerCluster(t *testing.T, ctx context.Con // would make it impossible to assert that ungranted brokers refuse to // rotate or remediate. if opts.grantDuration > 0 { - deadline := strconv.FormatInt(time.Now().Add(opts.grantDuration).Unix(), 10) + deadline := time.Now().Add(opts.grantDuration) for _, b := range brokers { require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(b), b)) p := client.MergeFrom(b.DeepCopy()) @@ -850,7 +847,7 @@ func (s *BrokerControllerSuite) setupBrokerCluster(t *testing.T, ctx context.Con b.Annotations = map[string]string{} } checksum := b.Spec.PodTemplate.Annotations["config.redpanda.com/checksum"] - b.Annotations["operator.redpanda.com/roll-grant"] = checksum + "/" + deadline + b.Annotations["operator.redpanda.com/roll-grant"] = feature.FormatRollGrant(checksum, deadline) require.NoError(t, c.Patch(ctx, b, p)) } } From e158df202bc93c0b2771c00afba4ded6157c6b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Tue, 14 Jul 2026 18:08:53 +0200 Subject: [PATCH 29/29] test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Bojanowski