Shorten grace period from 30m to 5m.
Check to remove every second to get rid of deactivated instances sooner.
@@ -22,8 +22,9 @@ import java.util.logging.Logger;
*/
class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable {
- static final Duration GRACE_PERIOD = Duration.ofMinutes(30);
+ static final Duration GRACE_PERIOD = Duration.ofMinutes(5);
static final Duration UPDATE_PERIOD = Duration.ofMinutes(5);
+ static final Duration CHECK_PERIOD = Duration.ofSeconds(1);
private static final Logger log = Logger.getLogger(ContainerWatchdog.class.getName());
@@ -35,6 +36,7 @@ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable {
private ActiveContainer currentContainer;
private Instant currentContainerActivationTime;
private int numStaleContainers;
+ private Instant lastLogTime;
ContainerWatchdog() {
this(new ScheduledThreadPoolExecutor(
@@ -50,8 +52,8 @@ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable {
ContainerWatchdog(ScheduledExecutorService scheduler, Clock clock) {
this.scheduler = scheduler;
this.clock = clock;
- scheduler.scheduleAtFixedRate(
- this::monitorDeactivatedContainers, UPDATE_PERIOD.getSeconds(), UPDATE_PERIOD.getSeconds(), TimeUnit.SECONDS);
+ this.lastLogTime = clock.instant();
+ scheduler.scheduleAtFixedRate(this::monitorDeactivatedContainers, CHECK_PERIOD.getSeconds(), CHECK_PERIOD.getSeconds(), TimeUnit.SECONDS);
}
@Override
@@ -77,31 +79,43 @@ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable {
void onContainerActivation(ActiveContainer nextContainer) {
synchronized (monitor) {
if (currentContainer != null) {
- deactivatedContainers.add(
- new DeactivatedContainer(currentContainer, currentContainerActivationTime, clock.instant()));
+ deactivatedContainers.add(new DeactivatedContainer(currentContainer, currentContainerActivationTime, clock.instant()));
}
currentContainer = nextContainer;
currentContainerActivationTime = clock.instant();
}
}
+ private String removalMsg(DeactivatedContainer container) {
+ return String.format("Removing deactivated container: instance=%s, activated=%s, deactivated=%s",
+ container.instance, container.timeActivated, container.timeDeactivated);
+ }
+
+ private String regularMsg(DeactivatedContainer container, int refCount) {
+ return String.format(
+ "Deactivated container still alive: instance=%s, activated=%s, deactivated=%s, ref-count=%d",
+ container.instance, container.timeActivated, container.timeDeactivated, refCount);
+ }
+
void monitorDeactivatedContainers() {
synchronized (monitor) {
int numStaleContainer = 0;
Iterator<DeactivatedContainer> iterator = deactivatedContainers.iterator();
+ boolean timeToLogAgain = clock.instant().isAfter(lastLogTime.plus(UPDATE_PERIOD));
while (iterator.hasNext()) {
DeactivatedContainer container = iterator.next();
int refCount = container.instance.retainCount();
if (refCount == 0) {
+ log.fine(removalMsg(container));
iterator.remove();
- break;
- }
+ } else {
if (isPastGracePeriod(container)) {
++numStaleContainer;
- log.warning(
- String.format(
- "Deactivated container still alive: instance=%s, activated=%s, deactivated=%s, ref-count=%d",
- container.instance.toString(), container.timeActivated, container.timeDeactivated, refCount));
+ if (timeToLogAgain) {
+ log.warning(regularMsg(container, refCount));
+ lastLogTime = clock.instant();
+ }
+ }
}
}
this.numStaleContainers = numStaleContainer;
