Defer upgrades of active deployments in perf for up to one week
package com.yahoo.vespa.hosted.controller.maintenance;
import com.yahoo.component.Version;
+import com.yahoo.config.provision.Environment;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.Instance;
@@ -11,11 +12,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.deployment.Run;
import com.yahoo.vespa.hosted.controller.deployment.Versions;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
import com.yahoo.yolean.Exceptions;
import java.time.Duration;
import java.time.Instant;
+import java.util.Comparator;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
@@ -37,7 +40,8 @@ public class DeploymentUpgrader extends ControllerMaintainer {
AtomicInteger failures = new AtomicInteger();
Version targetPlatform = null; // Upgrade to the newest non-broken, deployable version.
- for (VespaVersion platform : controller().readVersionStatus().deployableVersions())
+ VersionStatus versionStatus = controller().readVersionStatus();
+ for (VespaVersion platform : versionStatus.deployableVersions())
if (platform.confidence().equalOrHigherThan(VespaVersion.Confidence.normal))
targetPlatform = platform.versionNumber();
@@ -63,6 +67,7 @@ public class DeploymentUpgrader extends ControllerMaintainer {
if ( ! deployment.version().isBefore(target.targetPlatform())) continue;
if ( ! isLikelyNightFor(job)) continue;
+ if (deployment.zone().environment() == Environment.perf && ! isIdleOrOutdated(deployment, job)) continue;
log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone());
attempts.incrementAndGet();
@@ -76,6 +81,14 @@ public class DeploymentUpgrader extends ControllerMaintainer {
return asSuccessFactor(attempts.get(), failures.get());
}
+ /** Returns whether query and feed metrics are ~zero, or currently platform has been deployed for a week. */
+ private boolean isIdleOrOutdated(Deployment deployment, JobId job) {
+ if (deployment.metrics().queriesPerSecond() <= 1 && deployment.metrics().writesPerSecond() <= 1) return true;
+ return controller().jobController().runs(job).descendingMap().values().stream()
+ .takeWhile(run -> run.versions().targetPlatform().equals(deployment.version()))
+ .anyMatch(run -> run.start().isBefore(controller().clock().instant().minus(Duration.ofDays(7))));
+ }
+
private boolean isLikelyNightFor(JobId job) {
int hour = hourOf(controller().clock().instant());
int[] runStarts = controller().jobController().jobStarts(job).stream()
