Only store paragraph text in content field
When using -storeTransformedDocs, the contents field had some malformed text (title wrapped with Optional[] and kicker) - the content should be the body text of the document.
@@ -92,27 +92,18 @@ public class WapoGenerator extends LuceneDocumentGenerator<WashingtonPostCollect
});
StringBuilder contentBuilder = new StringBuilder();
- contentBuilder.append(wapoDoc.getTitle()).append("\n\n");
wapoDoc.getObj().getContents().ifPresent(contents -> {
for (WashingtonPostObject.Content contentObj : contents) {
- if (contentObj == null) continue;
- if (contentObj.getType().isPresent() && contentObj.getContent().isPresent()) {
+ if (contentObj != null) {
contentObj.getType().ifPresent(type -> {
contentObj.getContent().ifPresent(content -> {
if (CONTENT_TYPE_TAG.contains(type)) {
- contentBuilder.append(removeTags(content)).append("\n");
- } else if (type.compareToIgnoreCase("kicker") == 0) {
- doc.add(new StringField(WapoField.KICKER.name, content, Field.Store.NO));
+ contentBuilder.append(removeTags(content)).append(" ");
}
});
});
}
- contentObj.getFullCaption().ifPresent(caption -> {
- String fullCaption = contentObj.getFullCaption().get();
- doc.add(new StringField(WapoField.FULL_CAPTION.name, fullCaption, Field.Store.NO));
- contentBuilder.append(removeTags(fullCaption)).append("\n");
- });
}
});
