Update to PreprocessorMain: new option '--sentence-split' is now available
@@ -6,6 +6,7 @@ import eu.modernmt.lang.LanguageDirection;
import eu.modernmt.model.Sentence;
import eu.modernmt.processing.Preprocessor;
import eu.modernmt.processing.ProcessingException;
+import eu.modernmt.processing.splitter.SentenceSplitter;
import org.apache.commons.cli.*;
import org.apache.commons.io.IOUtils;
@@ -28,6 +29,7 @@ public class PreprocessorMain {
Option skipTags = Option.builder().longOpt("no-tags").hasArg(false).required(false).build();
Option skipPlaceholders = Option.builder().longOpt("print-placeholders").hasArg(false).required(false).build();
Option keepSpaces = Option.builder().longOpt("original-spacing").hasArg(false).required(false).build();
+ Option sentenceSplit = Option.builder().longOpt("sentence-split").hasArg(false).required(false).build();
cliOptions = new Options();
cliOptions.addOption(sourceLanguage);
@@ -36,6 +38,7 @@ public class PreprocessorMain {
cliOptions.addOption(skipTags);
cliOptions.addOption(skipPlaceholders);
cliOptions.addOption(keepSpaces);
+ cliOptions.addOption(sentenceSplit);
}
public final LanguageDirection language;
@@ -43,6 +46,7 @@ public class PreprocessorMain {
public final boolean printPlaceholders;
public final boolean keepSpaces;
public final boolean batch;
+ public final boolean split;
public Args(String[] args) throws ParseException {
CommandLineParser parser = new DefaultParser();
@@ -55,6 +59,7 @@ public class PreprocessorMain {
printPlaceholders = cli.hasOption("print-placeholders");
keepSpaces = cli.hasOption("original-spacing");
batch = cli.hasOption("batch");
+ split = cli.hasOption("sentence-split");
}
}
@@ -76,31 +81,47 @@ public class PreprocessorMain {
output = new TokensOutputter(args.printTags, args.printPlaceholders);
if (args.batch)
- batchPreprocess(preprocessor, args.language, input, output);
+ batchPreprocess(preprocessor, args.language, input, output, args.split);
else
- interactivePreprocess(preprocessor, args.language, input, output);
+ interactivePreprocess(preprocessor, args.language, input, output, args.split);
} finally {
IOUtils.closeQuietly(preprocessor);
IOUtils.closeQuietly(output);
}
}
- private static void interactivePreprocess(Preprocessor preprocessor, LanguageDirection language, LineReader input, Outputter output) throws IOException, ProcessingException {
+ private static void interactivePreprocess(Preprocessor preprocessor, LanguageDirection language, LineReader input, Outputter output, boolean split) throws IOException, ProcessingException {
String line;
while ((line = input.readLine()) != null) {
Sentence sentence = preprocessor.process(language, line);
- output.write(sentence);
+ write(output, sentence, split);
}
}
- private static void batchPreprocess(Preprocessor preprocessor, LanguageDirection language, LineReader input, Outputter output) throws IOException, ProcessingException {
+ private static void batchPreprocess(Preprocessor preprocessor, LanguageDirection language, LineReader input, Outputter output, boolean split) throws IOException, ProcessingException {
BufferedLineReader bufferedReader = new BufferedLineReader(input);
String[] batch;
while ((batch = bufferedReader.readLines()) != null) {
Sentence[] tokenized = preprocessor.process(language, batch);
for (Sentence sentence : tokenized)
- output.write(sentence);
+ write(output, sentence, split);
+ }
+ }
+
+ private static void write(Outputter outputter, Sentence sentence, boolean split) throws IOException {
+ Sentence[] sentences = null;
+
+ if (split) {
+ SentenceSplitter splitter = new SentenceSplitter(sentence);
+ sentences = splitter.split();
+ }
+
+ if (sentences == null) {
+ outputter.write(sentence);
+ } else {
+ for (Sentence sentenceSplit : sentences)
+ outputter.write(sentenceSplit);
}
}
