Increased default max file size in fetcher to 51MB
@@ -40,7 +40,11 @@ public class FetcherFactory {
httpFetcher.setConnectionRequestTimeout(1*60*1000);
httpFetcher.setMaxConnectionsPerHost(1);
httpFetcher.setMaxRetryCount(config.getMaxRetryCount());
- httpFetcher.setDefaultMaxContentSize(10*1024*1024);
+
+ // sets maximum file size: download of files larger than this will be aborted
+ // (note: should NOT be smaller than 50MB if crawling sitemap files)
+ int defaultMaxContentSize = 51 * 1024 * 1024;
+ httpFetcher.setDefaultMaxContentSize(defaultMaxContentSize);
if(config.getValidMimeTypes() != null) {
for (String mimeTypes : config.getValidMimeTypes()) {
