fix use duplicate host conversion
@@ -146,12 +146,15 @@ public class WebFsIndexHelper {
}
}
+ final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
+
// set urls
split(urlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
if (!urlValue.startsWith("#") && fessConfig.isValidCrawlerWebProtocol(urlValue)) {
- crawler.addUrl(urlValue);
+ final String u = duplicateHostHelper.convert(urlValue);
+ crawler.addUrl(u);
if (logger.isInfoEnabled()) {
- logger.info("Target URL: " + urlValue);
+ logger.info("Target URL: " + u);
}
}
}));
