fix in AlignerFilterMain
@@ -15,6 +15,7 @@ import eu.modernmt.processing.ProcessingException;
import eu.modernmt.training.LazyWriterMultilingualCorpus;
import eu.modernmt.training.MultilingualCorpusMask;
import org.apache.commons.cli.*;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.logging.log4j.Level;
@@ -83,12 +84,16 @@ public class AlignerFilterMain {
if (bilingualCorpora.isEmpty())
throw new ParseException("Input path does not contains valid bilingual data");
- FastAlign aligner = new FastAlign(args.model);
- Preprocessor preprocessor = new Preprocessor();
+ FileUtils.deleteDirectory(args.outputRoot);
+ FileUtils.forceMkdir(args.outputRoot);
LanguagePair languagePair = new LanguagePair(args.sourceLanguage, args.targetLanguage);
LanguageIndex languageIndex = new LanguageIndex(languagePair);
+ FastAlign aligner = new FastAlign(args.model);
+ Preprocessor preprocessor = new Preprocessor();
+
+ try {
Filter filter = new Filter(languagePair, aligner, preprocessor);
for (MultilingualCorpus _corpus : bilingualCorpora) {
@@ -97,6 +102,9 @@ public class AlignerFilterMain {
filter.apply(corpus, output, args.threshold);
}
+ } finally {
+ IOUtils.closeQuietly(preprocessor);
+ }
}
private static final class Filter {
