Improved XMLStripper
@@ -3,43 +3,54 @@ package eu.modernmt.cleaning.normalizers;
import eu.modernmt.cleaning.Normalizer;
import eu.modernmt.model.Tag;
import eu.modernmt.model.corpus.MultilingualCorpus;
+import eu.modernmt.processing.xml.XMLCharacterEntity;
-import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* Created by davide on 17/11/16.
*/
public class XMLStripper implements Normalizer {
- private static String stripXML(String line) {
- StringBuilder builder = new StringBuilder();
- Matcher m = Tag.TagRegex.matcher(line);
+ public static final Pattern DoubleEntityPattern = Pattern.compile("&amp;((#[0-9]{1,4})|(#x[0-9a-fA-F]{1,4})|([a-zA-Z]+));");
- int stringIndex = 0;
- char[] chars = line.toCharArray();
+ private static String stripXML(String line) {
+ return Tag.TagRegex.matcher(line).replaceAll(" ");
+ }
- while (m.find()) {
- int start = m.start();
- int end = m.end();
+ private static String stripDoubleEscapedEntities(String line) {
+ return DoubleEntityPattern.matcher(line).replaceAll(" ");
+ }
- if (stringIndex < start)
- builder.append(chars, stringIndex, start - stringIndex);
+ private static boolean hasXMLEntities(String line) {
+ return XMLCharacterEntity.EntityPattern.matcher(line).find();
+ }
- stringIndex = end;
+ private static String normalize(String line) {
+ while (true) {
+ // if it has valid XML then we stop: if you put actual XML all subsequent entities should be intentional
+ String plain = stripXML(line);
+ if (!plain.equals(line))
+ return plain;
- builder.append(' ');
- }
+ // if there are no double-escaped entities (i.e. &amp;lt;), problem is solved!
+ String stripped = stripDoubleEscapedEntities(line);
+ if (stripped.equals(line))
+ return line;
- if (stringIndex < chars.length)
- builder.append(chars, stringIndex, chars.length - stringIndex);
+ // if, excluding double-escaped entities, we found entities, it means that the first ones were intentional
+ if (hasXMLEntities(stripped))
+ return line;
- return builder.toString();
+ // if we only have double-escaped entities, we suppose those were generated by accident
+ line = XMLCharacterEntity.unescapeAll(line);
+ }
}
@Override
public void normalize(MultilingualCorpus.StringPair pair, int index) {
- pair.source = stripXML(pair.source).trim();
- pair.target = stripXML(pair.target).trim();
+ pair.source = normalize(pair.source).trim();
+ pair.target = normalize(pair.target).trim();
}
}
