Check if page is html before try to get html metadata
@@ -30,11 +30,12 @@ public class TargetModelElasticSearch {
}
public TargetModelElasticSearch(Page page) {
-
this.url = page.getURL().toString();
this.retrieved = new Date();
this.domain = page.getDomainName();
this.html = page.getContentAsString();
+ this.topPrivateDomain = LinkRelevance.getTopLevelDomain(page.getDomainName());
+ if (page.isHtml()) {
this.words = page.getParsedData().getWords();
this.wordsMeta = page.getParsedData().getWordsMeta();
this.title = page.getParsedData().getTitle();
@@ -45,7 +46,7 @@ public class TargetModelElasticSearch {
} catch (BoilerpipeProcessingException | ArrayIndexOutOfBoundsException e) {
this.text = "";
}
- this.topPrivateDomain = LinkRelevance.getTopLevelDomain(page.getDomainName());
+ }
}
public TargetModelElasticSearch(TargetModelCbor model) {
