Add more judge between Charset and ISO-8859-1(5)
Judge if content has hex value which is nonexistent in Charset
@@ -59,7 +59,8 @@ class UniversalEncodingListener implements CharsetListener {
if (hint != null) {
// Use the encoding hint when available
name = hint;
- } else if (statistics.count('\r') == 0) {
+ } else if (hasNonexistentHexInCharsetWindows1252() || statistics.count('\r') == 0) {
+ // If it has nonexistent hex value in charset windows-1252 or
// If there are no CR(LF)s, then the encoding is more
// likely to be ISO-8859-1(5) than windows-1252
if (statistics.count(0xa4) > 0) { // currency/euro sign
@@ -97,4 +98,16 @@ class UniversalEncodingListener implements CharsetListener {
return charset;
}
+ /*
+ * hex value 0x81, 0x8d, 0x8f, 0x90, 0x9d don't exist in charset windows-1252.
+ * If these value's count > 0, return true
+ * */
+ private Boolean hasNonexistentHexInCharsetWindows1252() {
+ return (statistics.count(0x81) > 0 ||
+ statistics.count(0x8d) > 0 ||
+ statistics.count(0x8f) > 0 ||
+ statistics.count(0x90) > 0 ||
+ statistics.count(0x9d) > 0);
+ }
+
}
\ No newline at end of file
