fix remove distinct
@@ -161,8 +161,7 @@ public class WebFsIndexHelper {
}));
// set included urls
- split(includedUrlsStr, "[\r\n]").of(
- stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
+ split(includedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(urlValue -> {
if (!urlValue.startsWith("#")) {
crawler.addIncludeFilter(urlValue);
if (logger.isInfoEnabled()) {
@@ -172,8 +171,7 @@ public class WebFsIndexHelper {
}));
// set excluded urls
- split(excludedUrlsStr, "[\r\n]").of(
- stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
+ split(excludedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(urlValue -> {
if (!urlValue.startsWith("#")) {
crawler.addExcludeFilter(urlValue);
if (logger.isInfoEnabled()) {
@@ -277,8 +275,7 @@ public class WebFsIndexHelper {
// set included paths
final AtomicBoolean urlEncodeDisabled = new AtomicBoolean(false);
- split(includedPathsStr, "[\r\n]").of(
- stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(line -> {
+ split(includedPathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
if (!line.startsWith("#")) {
final String urlValue;
if (urlEncodeDisabled.get()) {
@@ -298,8 +295,7 @@ public class WebFsIndexHelper {
// set excluded paths
urlEncodeDisabled.set(false);
- split(excludedPathsStr, "[\r\n]").of(
- stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(line -> {
+ split(excludedPathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
if (!line.startsWith("#")) {
final String urlValue;
if (urlEncodeDisabled.get()) {
