Fixed TypeError when launching Neural training without GPUs
@@ -352,7 +352,6 @@ class NeuralEngineBuilder(EngineBuilder):
else os.path.join(self._engine.data_path, TrainingPreprocessor.DEV_FOLDER_NAME)
self._checkpoint = checkpoint
self._metadata = metadata
- self._gpus = gpus
def _build_schedule(self):
return EngineBuilder._build_schedule(self) + \
@@ -361,13 +360,13 @@ class NeuralEngineBuilder(EngineBuilder):
def _check_constraints(self):
recommended_gpu_ram = 2 * self._GB
- # if the user explicitly said that no GPU must be used, return immediately
- if self._gpus == [-1]:
- return
-
- # else, get the list of GPUs to employ using torch utils (This takes into account the user's choice)
+ # Get the list of GPUs to employ using torch utils (This takes into account the user's choice)
gpus = torch_utils.torch_get_gpus()
+ if gpus is None or len(gpus) == 0:
+ raise EngineBuilder.HWConstraintViolated(
+ 'No GPU for Neural engine training, the process will take very long time to complete.')
+
# AT THE MOMENT TRAINING IS MONOGPU AND WE ONLY USE THE FIRST AVAILABLE GPU FOR TRAINING.
# SO JUST CHECK CONSTRAINTS FOR IT. THIS MAY CHANGE IN THE FUTURE
gpus = [gpus[0]]
@@ -383,7 +382,8 @@ class NeuralEngineBuilder(EngineBuilder):
def _get_gpus_ram(self, gpu_ids):
result = []
- command = ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits", "--id=%s" % ",".join(str(id) for id in gpu_ids)]
+ command = ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits",
+ "--id=%s" % ",".join(str(i) for i in gpu_ids)]
stdout, _ = shell.execute(command)
for line in stdout.split("\n"):
line = line.strip()
