Improved KnownEntities post-processor
@@ -13,8 +13,11 @@ import eu.modernmt.processing.detokenizer.jflex.JFlexSpaceAnnotator;
/* Default Ignore */
. { /* ignore */ }
+/* Line separators |||| */
+(\|{_}?)(\|{_}?)+ { return REMOVE_INNER; }
+
/* Variable-like strings such as: ${example} or %{var_1} */
-{_}[\$\%]{_}?\{{_}?[a-zA-Z0-9_]+{_}?\}{_} { return REMOVE_INNER; }
+[\$\%]{_}?\{{_}?([a-zA-Z0-9_ ])+{_}?\} { return REMOVE_INNER; }
/* String formatting variables: https://developer.android.com/reference/java/util/Formatter */
-{_}(\%){_}?([0-9]{_}?\$+)?{_}?([\-#+0,\(])?{_}?([0-9]+)?{_}?(\.[0-9]+)?{_}?([bBhHsScCdoxXeEfgGaA\%n]|[t|T][BbhAaCYyjmdeRTrDFc]){_} { return REMOVE_INNER; }
\ No newline at end of file
+(\%){_}?([0-9]{_}?\$+)?{_}?([\-#+0,\(])?{_}?([0-9]+)?{_}?(\.[0-9]+)?{_}?([bBhHsScCdoxXeEfgGaA\%n]|[t|T][BbhAaCYyjmdeRTrDFc])[^a-zA-Z] { return REMOVE_INNER; }
\ No newline at end of file
