add prefix to S3Fetcher
@@ -45,6 +45,7 @@ import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
import org.apache.tika.pipes.fetcher.RangeFetcher;
+import org.apache.tika.utils.StringUtils;
/**
* Fetches files from s3. Example string: s3://my_bucket/path/to/my_file.pdf
@@ -58,6 +59,7 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
private String region;
private String bucket;
private String profile;
+ private String prefix;
private String credentialsProvider;
private boolean extractUserMetadata = true;
private int maxConnections = ClientConfiguration.DEFAULT_MAX_CONNECTIONS;
@@ -67,10 +69,11 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
@Override
public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
- LOGGER.debug("about to fetch fetchkey={} from bucket ({})", fetchKey, bucket);
+ String theFetchKey = StringUtils.isBlank(prefix) ? fetchKey : prefix + fetchKey;
+ LOGGER.debug("about to fetch fetchkey={} from bucket ({})", theFetchKey, bucket);
try {
- S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, fetchKey));
+ S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, theFetchKey));
if (extractUserMetadata) {
for (Map.Entry<String, String> e : s3Object.getObjectMetadata().getUserMetadata()
.entrySet()) {
@@ -95,13 +98,14 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
@Override
public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
throws TikaException, IOException {
+ String theFetchKey = StringUtils.isBlank(prefix) ? fetchKey : prefix + fetchKey;
//TODO -- figure out how to integrate this
- LOGGER.debug("about to fetch fetchkey={} (start={} end={}) from bucket ({})", fetchKey,
- startRange, endRange, bucket);
+ LOGGER.debug("about to fetch fetchkey={} (start={} end={}) from bucket ({})",
+ theFetchKey, startRange, endRange, bucket);
try {
S3Object s3Object = s3Client.getObject(
- new GetObjectRequest(bucket, fetchKey).withRange(startRange, endRange));
+ new GetObjectRequest(bucket, theFetchKey).withRange(startRange, endRange));
if (extractUserMetadata) {
for (Map.Entry<String, String> e : s3Object.getObjectMetadata().getUserMetadata()
@@ -144,6 +148,19 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
this.bucket = bucket;
}
+ /**
+ * prefix to prepend to the fetch key before fetching.
+ * This will automatically add a '/' at the end.
+ * @param prefix
+ */
+ @Field
+ public void setPrefix(String prefix) {
+ //guarantee that the prefix ends with /
+ if (! prefix.endsWith("/")) {
+ prefix += "/";
+ }
+ this.prefix = prefix;
+ }
/**
* Whether or not to extract user metadata from the S3Object
*
