Improve timeout and retry handling when failing to reconfigure ZK
@@ -29,7 +29,7 @@ public class Reconfigurer extends AbstractComponent {
private static final Logger log = java.util.logging.Logger.getLogger(Reconfigurer.class.getName());
- private static final Duration TIMEOUT = Duration.ofMinutes(3);
+ private static final Duration TIMEOUT = Duration.ofMinutes(15);
private final ExponentialBackoff backoff = new ExponentialBackoff(Duration.ofSeconds(1), Duration.ofSeconds(10));
private final VespaZooKeeperAdmin vespaZooKeeperAdmin;
@@ -97,7 +97,7 @@ public class Reconfigurer extends AbstractComponent {
Duration reconfigTimeout = reconfigTimeout();
Instant end = now.plus(reconfigTimeout);
// Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed
- for (int attempt = 1; now.isBefore(end); attempt++) {
+ for (int attempt = 1; ; attempt++) {
try {
Instant reconfigStarted = Instant.now();
vespaZooKeeperAdmin.reconfigure(connectionSpec, newServers);
@@ -110,18 +110,20 @@ public class Reconfigurer extends AbstractComponent {
return;
} catch (ReconfigException e) {
Duration delay = backoff.delay(attempt);
+ now = Instant.now();
+ if (now.isBefore(end)) {
log.log(Level.WARNING, "Reconfiguration attempt " + attempt + " failed. Retrying in " + delay +
- ", time left " + Duration.between(now, end) + ": " +
- Exceptions.toMessageString(e));
+ ", time left " + Duration.between(now, end) + ": " + Exceptions.toMessageString(e));
sleeper.sleep(delay);
- } finally {
- now = Instant.now();
}
- }
-
- // Reconfiguration failed
+ else {
+ log.log(Level.SEVERE, "Reconfiguration attempt " + attempt + " failed, and was failing for " +
+ reconfigTimeout + "; giving up now: " + Exceptions.toMessageString(e));
shutdownAndDie(reconfigTimeout);
}
+ }
+ }
+ }
private void shutdownAndDie(Duration reconfigTimeout) {
shutdown();
