Support runtime configuration of maxWaitTimeInMinutes

CHANGES.md

@@ -6,6 +6,7 @@
 * Add JSON output format for ecctool state - Issue #1739
 * Fix repair jobs stuck ON_TIME when Jolokia returns empty body due to transient network issues - Issue #1740
 * Fix Jolokia JMX connection health check always returning true for stale connections - Issue #1764
+* Support runtime configuration of maxWaitTimeInMinutes via ecctool config - Issue #1783
 
 ## Version 1.0.6
 

application/src/main/java/com/ericsson/bss/cassandra/ecchronos/application/spring/ECChronos.java

@@ -49,6 +49,7 @@
 import com.ericsson.bss.cassandra.ecchronos.core.impl.repair.vnode.VnodeRepairStateFactoryImpl;
 import com.ericsson.bss.cassandra.ecchronos.core.impl.table.TimeBasedRunPolicy;
 import com.ericsson.bss.cassandra.ecchronos.core.repair.RepairStatsProvider;
+import com.ericsson.bss.cassandra.ecchronos.core.jmx.DistributedJmxProxyFactory;
 import com.ericsson.bss.cassandra.ecchronos.core.repair.scheduler.OnDemandRepairScheduler;
 import com.ericsson.bss.cassandra.ecchronos.core.repair.scheduler.ScheduleManager;
 import com.ericsson.bss.cassandra.ecchronos.core.repair.scheduler.RepairScheduler;
@@ -262,6 +263,18 @@ public ScheduleManager scheduleManager()
         return myECChronosInternals.getScheduleManager();
     }
 
+    /**
+     * Returns the distributed JMX proxy factory, exposed for runtime configuration
+     * of the repair max wait time.
+     *
+     * @return the {@link DistributedJmxProxyFactory} instance.
+     */
+    @Bean
+    public DistributedJmxProxyFactory jmxProxyFactory()
+    {
+        return myECChronosInternals.getJmxProxyFactory();
+    }
+
     /**
      * Returns the repair statistics provider for querying repair stats.
      *

application/src/test/java/com/ericsson/bss/cassandra/ecchronos/application/spring/TestTomcatWebServerCustomizer.java

@@ -54,6 +54,7 @@
 
 import com.ericsson.bss.cassandra.ecchronos.application.utils.CertUtils;
 import com.ericsson.bss.cassandra.ecchronos.connection.DistributedJmxConnectionProvider;
+import com.ericsson.bss.cassandra.ecchronos.core.jmx.DistributedJmxProxyFactory;
 import com.ericsson.bss.cassandra.ecchronos.connection.DistributedNativeConnectionProvider;
 import com.ericsson.bss.cassandra.ecchronos.core.impl.table.TimeBasedRunPolicy;
 import com.ericsson.bss.cassandra.ecchronos.core.metadata.NodeResolver;
@@ -130,6 +131,9 @@ public abstract class TestTomcatWebServerCustomizer
     @MockitoBean
     private ScheduleManager scheduleManager;
 
+    @MockitoBean
+    private DistributedJmxProxyFactory jmxProxyFactory;
+
     @MockitoBean
     private DistributedNativeConnectionProvider nativeConnectionProvider;
 

core.impl/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/impl/jmx/DistributedJmxProxyFactoryImpl.java

@@ -37,7 +37,7 @@ public final class  DistributedJmxProxyFactoryImpl implements DistributedJmxProx
     private final Map<UUID, Node> nodesMap;
     private final EccNodesSync eccNodesSync;
     private final boolean isJolokiaEnabled;
-    private final Integer myMaxWaitTimeInMinutes;
+    private volatile Integer myMaxWaitTimeInMinutes;
     private final JolokiaNotificationController myJolokiaNotificationController;
 
     private DistributedJmxProxyFactoryImpl(final Builder builder)
@@ -88,6 +88,19 @@ public Integer getMaxWaitTimeInMinutes()
         return myMaxWaitTimeInMinutes;
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void setMaxWaitTimeInMinutes(final int maxWaitTimeInMinutes)
+    {
+        if (maxWaitTimeInMinutes <= 0)
+        {
+            throw new IllegalArgumentException("maxWaitTimeInMinutes must be > 0");
+        }
+        myMaxWaitTimeInMinutes = maxWaitTimeInMinutes;
+    }
+
     /**
      * Creates a new builder for constructing {@link DistributedJmxProxyFactoryImpl} instances.
      *

core.impl/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/impl/jmx/TestDistributedJmxProxyFactoryImpl.java

@@ -240,4 +240,59 @@ public void testIsRepairActiveExceptionReturnsTrue() throws Exception
 
         assertTrue(distributedJmxProxy.isRepairActive(nodeId, 42));
     }
+
+    @Test
+    public void testDefaultMaxWaitTimeInMinutes()
+    {
+        DistributedJmxProxyFactoryImpl factory = DistributedJmxProxyFactoryImpl.builder()
+                .withJmxConnectionProvider(mockConnectionProvider)
+                .withNodesMap(mockNodesMap)
+                .withIpTranslator(new IpTranslator())
+                .withEccNodesSync(mockEccNodesSync)
+                .build();
+
+        assertEquals(Integer.valueOf(DistributedJmxProxyFactoryImpl.Builder.DEFAULT_MAX_WAIT_TIME_IN_MINUTES),
+                factory.getMaxWaitTimeInMinutes());
+    }
+
+    @Test
+    public void testSetMaxWaitTimeInMinutesUpdatesValue()
+    {
+        DistributedJmxProxyFactoryImpl factory = DistributedJmxProxyFactoryImpl.builder()
+                .withJmxConnectionProvider(mockConnectionProvider)
+                .withNodesMap(mockNodesMap)
+                .withIpTranslator(new IpTranslator())
+                .withEccNodesSync(mockEccNodesSync)
+                .build();
+
+        factory.setMaxWaitTimeInMinutes(60);
+
+        assertEquals(Integer.valueOf(60), factory.getMaxWaitTimeInMinutes());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetMaxWaitTimeInMinutesRejectsZero()
+    {
+        DistributedJmxProxyFactoryImpl factory = DistributedJmxProxyFactoryImpl.builder()
+                .withJmxConnectionProvider(mockConnectionProvider)
+                .withNodesMap(mockNodesMap)
+                .withIpTranslator(new IpTranslator())
+                .withEccNodesSync(mockEccNodesSync)
+                .build();
+
+        factory.setMaxWaitTimeInMinutes(0);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetMaxWaitTimeInMinutesRejectsNegative()
+    {
+        DistributedJmxProxyFactoryImpl factory = DistributedJmxProxyFactoryImpl.builder()
+                .withJmxConnectionProvider(mockConnectionProvider)
+                .withNodesMap(mockNodesMap)
+                .withIpTranslator(new IpTranslator())
+                .withEccNodesSync(mockEccNodesSync)
+                .build();
+
+        factory.setMaxWaitTimeInMinutes(-5);
+    }
 }

core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/jmx/DistributedJmxProxyFactory.java

@@ -37,5 +37,15 @@ public interface DistributedJmxProxyFactory
      * @return The maximum wait time in minutes.
      */
     Integer getMaxWaitTimeInMinutes();
+
+    /**
+     * Set the maximum wait time in minutes for repair tasks at runtime.
+     * <p>
+     * New repair tasks pick up the new value on creation; in-flight repairs keep
+     * the value captured when they started.
+     *
+     * @param maxWaitTimeInMinutes The maximum wait time in minutes. Must be greater than 0.
+     */
+    void setMaxWaitTimeInMinutes(int maxWaitTimeInMinutes);
 }
 

docs/ECCTOOL_EXAMPLES.md

@@ -523,7 +523,8 @@ $ ecctool config
 {
   "session_window_ms": 300000,
   "cooldown_ms": 0,
-  "locks_per_resource": 3
+  "locks_per_resource": 3,
+  "max_wait_time_minutes": 40
 }
 ```
 
@@ -534,21 +535,35 @@ $ ecctool config --session-window 10m
 {
   "session_window_ms": 600000,
   "cooldown_ms": 0,
-  "locks_per_resource": 3
+  "locks_per_resource": 3,
+  "max_wait_time_minutes": 40
+}
+```
+
+### Update the repair max wait time to 60 minutes
+
+```console
+$ ecctool config --max-wait-time 60
+{
+  "session_window_ms": 300000,
+  "cooldown_ms": 0,
+  "locks_per_resource": 3,
+  "max_wait_time_minutes": 60
 }
 ```
 
 ### Update multiple parameters
 
 ```console
-$ ecctool config --session-window 5m --cooldown 30s --locks-per-resource 5
+$ ecctool config --session-window 5m --cooldown 30s --locks-per-resource 5 --max-wait-time 60
 {
   "session_window_ms": 300000,
   "cooldown_ms": 30000,
-  "locks_per_resource": 5
+  "locks_per_resource": 5,
+  "max_wait_time_minutes": 60
 }
 ```
 
-Duration values accept: `5m` (minutes), `30s` (seconds), `2h` (hours), `500ms` (milliseconds), or raw milliseconds as integers.
+Duration values accept: `5m` (minutes), `30s` (seconds), `2h` (hours), `500ms` (milliseconds), or raw milliseconds as integers. The `--max-wait-time` value is in minutes and must be greater than 0.
 
 Changes are in-memory only. Restarting ecChronos restores values from `ecc.yml`.

docs/autogenerated/ECCTOOL.md

@@ -489,7 +489,7 @@ file containing process id
 Show or update ecChronos runtime configuration. Changes are in-memory only — restarting ecChronos restores values from `ecc.yml`.
 
 ```console
-usage: ecctool config [-h] [--session-window SESSION_WINDOW] [--cooldown COOLDOWN] [--locks-per-resource LOCKS_PER_RESOURCE] [-u URL]
+usage: ecctool config [-h] [--session-window SESSION_WINDOW] [--cooldown COOLDOWN] [--locks-per-resource LOCKS_PER_RESOURCE] [--max-wait-time MAX_WAIT_TIME] [-u URL]
 ```
 
 When called without arguments, displays the current configuration. When called with one or more parameters, updates the specified values.
@@ -503,6 +503,9 @@ Cooldown period after a session completes. Accepts same duration format as `--se
 ### --locks-per-resource <int>
 Number of concurrent locks per datacenter resource. Must be >= 1.
 
+### --max-wait-time <int>
+Maximum time in minutes ecChronos waits for a repair to complete before terminating and rescheduling it. Must be > 0. New repairs pick up the value immediately; in-flight repairs keep their original timeout.
+
 ### -u <url>, --url <url>
 ecchronos host URL (format: [http:/](http:/)/<host>:<port>)
 

ecchronos-binary/src/bin/ecctool.py

@@ -208,6 +208,7 @@ def add_config_subcommand(sub_parsers):
     parser_config.add_argument("--session-window", type=str, help="session window duration (e.g. 5m, 30s, 300000)")
     parser_config.add_argument("--cooldown", type=str, help="cooldown duration (e.g. 5m, 30s, 300000)")
     parser_config.add_argument("--locks-per-resource", type=int, help="locks per resource")
+    parser_config.add_argument("--max-wait-time", type=int, help="max wait time in minutes for a repair (> 0)")
     add_common_arg(parser_config, ARG_URL)
 
 
@@ -217,6 +218,7 @@ def config(arguments):
         arguments.session_window is not None
         or arguments.cooldown is not None
         or arguments.locks_per_resource is not None
+        or arguments.max_wait_time is not None
     )
     if has_updates:
         session_window_ms = parse_duration_ms(arguments.session_window) if arguments.session_window else None
@@ -225,6 +227,7 @@ def config(arguments):
             session_window_ms=session_window_ms,
             cooldown_ms=cooldown_ms,
             locks_per_resource=arguments.locks_per_resource,
+            max_wait_time_minutes=arguments.max_wait_time,
         )
     else:
         result = request.get()

ecchronos-binary/src/pylib/ecchronoslib/rest.py

@@ -389,14 +389,16 @@ def __init__(self, base_url=None):
     def get(self):
         return self.request(ConfigRequest.URL)
 
-    def patch(self, session_window_ms=None, cooldown_ms=None, locks_per_resource=None):
+    def patch(self, session_window_ms=None, cooldown_ms=None, locks_per_resource=None, max_wait_time_minutes=None):
         body = {}
         if session_window_ms is not None:
             body["session_window_ms"] = session_window_ms
         if cooldown_ms is not None:
             body["cooldown_ms"] = cooldown_ms
         if locks_per_resource is not None:
             body["locks_per_resource"] = locks_per_resource
+        if max_wait_time_minutes is not None:
+            body["max_wait_time_minutes"] = max_wait_time_minutes
         headers = {"Content-Type": "application/json"}
         return self.request(ConfigRequest.URL, "PATCH", body=body, headers=headers)
 

rest/src/main/java/com/ericsson/bss/cassandra/ecchronos/rest/ConfigManagementRESTImpl.java

@@ -15,6 +15,7 @@
 package com.ericsson.bss.cassandra.ecchronos.rest;
 
 import com.ericsson.bss.cassandra.ecchronos.core.repair.scheduler.ScheduleManager;
+import com.ericsson.bss.cassandra.ecchronos.core.jmx.DistributedJmxProxyFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
@@ -38,19 +39,25 @@ public final class ConfigManagementRESTImpl
     private static final String KEY_SESSION_WINDOW = "session_window_ms";
     private static final String KEY_COOLDOWN = "cooldown_ms";
     private static final String KEY_LOCKS_PER_RESOURCE = "locks_per_resource";
+    private static final String KEY_MAX_WAIT_TIME = "max_wait_time_minutes";
     private static final int MIN_LOCKS_PER_RESOURCE = 1;
+    private static final int MIN_MAX_WAIT_TIME = 1;
 
     private final ScheduleManager myScheduleManager;
+    private final DistributedJmxProxyFactory myJmxProxyFactory;
 
     /**
      * Constructs the configuration management REST controller.
      *
      * @param scheduleManager the schedule manager providing configuration access.
+     * @param jmxProxyFactory the JMX proxy factory providing the repair max wait time.
      */
     @Autowired
-    public ConfigManagementRESTImpl(final ScheduleManager scheduleManager)
+    public ConfigManagementRESTImpl(final ScheduleManager scheduleManager,
+            final DistributedJmxProxyFactory jmxProxyFactory)
     {
         myScheduleManager = scheduleManager;
+        myJmxProxyFactory = jmxProxyFactory;
     }
 
     /**
@@ -92,6 +99,7 @@ private void validatePatchBody(final Map<String, Object> body)
         validateMin(body, KEY_SESSION_WINDOW, 1, "session_window must be > 0");
         validateMin(body, KEY_COOLDOWN, 0, "cooldown must be >= 0");
         validateMin(body, KEY_LOCKS_PER_RESOURCE, MIN_LOCKS_PER_RESOURCE, "locks_per_resource must be >= 1");
+        validateMin(body, KEY_MAX_WAIT_TIME, MIN_MAX_WAIT_TIME, "max_wait_time_minutes must be > 0");
     }
 
     private void validateMin(final Map<String, Object> body, final String key, final long min, final String message)
@@ -116,6 +124,10 @@ private void applyPatchBody(final Map<String, Object> body)
         {
             myScheduleManager.setLocksPerResource(((Number) body.get(KEY_LOCKS_PER_RESOURCE)).intValue());
         }
+        if (body.containsKey(KEY_MAX_WAIT_TIME))
+        {
+            myJmxProxyFactory.setMaxWaitTimeInMinutes(((Number) body.get(KEY_MAX_WAIT_TIME)).intValue());
+        }
     }
 
     private Map<String, Object> buildResponse()
@@ -124,6 +136,7 @@ private Map<String, Object> buildResponse()
         config.put(KEY_SESSION_WINDOW, myScheduleManager.getSessionWindowInMs());
         config.put(KEY_COOLDOWN, myScheduleManager.getCooldownInMs());
         config.put(KEY_LOCKS_PER_RESOURCE, myScheduleManager.getLocksPerResource());
+        config.put(KEY_MAX_WAIT_TIME, myJmxProxyFactory.getMaxWaitTimeInMinutes());
         return config;
     }
 }

rest/src/test/java/com/ericsson/bss/cassandra/ecchronos/rest/ITConfigManagement.java

@@ -20,9 +20,14 @@
 
 import com.datastax.oss.driver.api.core.metadata.Node;
 import com.ericsson.bss.cassandra.ecchronos.connection.DistributedNativeConnectionProvider;
+import com.ericsson.bss.cassandra.ecchronos.connection.DistributedJmxConnectionProvider;
+import com.ericsson.bss.cassandra.ecchronos.core.impl.jmx.DistributedJmxProxyFactoryImpl;
 import com.ericsson.bss.cassandra.ecchronos.core.impl.locks.CASLockFactory;
 import com.ericsson.bss.cassandra.ecchronos.core.impl.repair.RepairLockFactoryImpl;
 import com.ericsson.bss.cassandra.ecchronos.core.impl.repair.scheduler.ScheduleManagerImpl;
+import com.ericsson.bss.cassandra.ecchronos.core.jmx.DistributedJmxProxyFactory;
+import com.ericsson.bss.cassandra.ecchronos.data.iptranslator.IpTranslator;
+import com.ericsson.bss.cassandra.ecchronos.data.sync.EccNodesSync;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -41,6 +46,7 @@ public class ITConfigManagement
 {
     private ConfigManagementRESTImpl myController;
     private ScheduleManagerImpl myScheduleManager;
+    private DistributedJmxProxyFactory myJmxProxyFactory;
     private int originalLocksPerResource;
 
     @Before
@@ -58,7 +64,14 @@ public void setup()
                 .withLockFactory(lockFactory)
                 .build();
 
-        myController = new ConfigManagementRESTImpl(myScheduleManager);
+        myJmxProxyFactory = DistributedJmxProxyFactoryImpl.builder()
+                .withJmxConnectionProvider(mock(DistributedJmxConnectionProvider.class))
+                .withNodesMap(Map.of(nodeId, mockNode))
+                .withIpTranslator(new IpTranslator())
+                .withEccNodesSync(mock(EccNodesSync.class))
+                .build();
+
+        myController = new ConfigManagementRESTImpl(myScheduleManager, myJmxProxyFactory);
         originalLocksPerResource = RepairLockFactoryImpl.getLocksPerResource();
     }
 
@@ -79,6 +92,19 @@ public void testGetReturnsCurrentConfig()
         assertThat(((Number) body.get("session_window_ms")).longValue()).isEqualTo(300000L);
         assertThat(((Number) body.get("cooldown_ms")).longValue()).isEqualTo(0L);
         assertThat(((Number) body.get("locks_per_resource")).intValue()).isEqualTo(originalLocksPerResource);
+        assertThat(((Number) body.get("max_wait_time_minutes")).intValue()).isEqualTo(40);
+    }
+
+    @Test
+    public void testPatchMaxWaitTimeRoundTrip()
+    {
+        Map<String, Object> patch = new HashMap<>();
+        patch.put("max_wait_time_minutes", 75);
+
+        ResponseEntity<Map<String, Object>> response = myController.patchConfig(patch);
+
+        assertThat(((Number) response.getBody().get("max_wait_time_minutes")).intValue()).isEqualTo(75);
+        assertThat(myJmxProxyFactory.getMaxWaitTimeInMinutes()).isEqualTo(75);
     }
 
     @Test