bug fixed: correct handling of subtoken '_'
@@ -115,10 +115,19 @@ class ModernMTSubwordTextEncoder(SubwordTextEncoder):
def _get_indexes(subtokens):
indexes = []
i = 0
+ final = True
for subtoken in subtokens:
+ if subtoken == "_" and final: # handle the subtoken containing only the EndOfToken. associate it with the same previous token (if final)
+ i = i - 1 if i > 0 else 0
+
indexes.append(i)
+
if subtoken.endswith('_'):
i += 1
+ final = True
+ else:
+ final = False
+
return indexes
