bug fixed: termination policy computed on the same checkpoints used for final model average
@@ -67,6 +67,7 @@ class TrainActivity(StatefulActivity):
self.state.step_no = self._index_of_step('train_nn') - 1
self.state.warmup_updates = int(argv_valueof(extra_argv, '--warmup-updates'))
+ self.state.save_interval_updates = int(argv_valueof(extra_argv, '--save-interval-updates'))
def _training_should_stop(self):
valid_folder = os.path.join(self.state.tensorboard_logdir, 'valid')
@@ -84,6 +85,10 @@ class TrainActivity(StatefulActivity):
if step < self.state.warmup_updates * 2:
return False
+ # if event does not correspond to a storable checkpoint, skip
+ if step % self.state.self.state.save_interval_updates != 0:
+ return False
+
window = self.args.num_checkpoints
current_loss = sum([_get_loss(e)[1] for e in events[:window]]) / window
previous_loss = sum([_get_loss(e)[1] for e in events[1:window + 1]]) / window
