fix(manager): work around elasticsearch#19862 index race condition
Can be a race condition to create an ES index (such as apiman_metrics) which may cause secondary
attempts to create the index to fail. We just need to ignore this as there's still no
'create if not exist' in ES.
See:
Fixes:
@@ -29,6 +29,7 @@ import java.util.Set;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -38,6 +39,7 @@ import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestStatus;
/**
* Base class for client factories. Provides caching of clients.
@@ -155,7 +157,7 @@ public abstract class AbstractClientFactory {
* @throws IOException if problem connecting to or interacting with ES
*/
@SuppressWarnings("nls")
- protected void createIndex(RestHighLevelClient client, EsIndexProperties indexDef, String indexPrefix, String indexPostfix) throws Exception {
+ protected synchronized void createIndex(RestHighLevelClient client, EsIndexProperties indexDef, String indexPrefix, String indexPostfix) throws Exception {
String indexToCreate = EsIndexMapping.getFullIndexName(indexPrefix, indexPostfix);
CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexToCreate);
String mapping;
@@ -164,11 +166,11 @@ public abstract class AbstractClientFactory {
// Create index using a full definition.
createIndexRequest = createIndexRequest.mapping(mapping, XContentType.JSON);
} catch (JsonProcessingException jpe) {
- logger.warn("The EsIndex definition provided by {} definition cannot be marshalled", this.getClass().getCanonicalName());
+ logger.warn("The EsIndex definition provided by {0} definition cannot be marshalled", this.getClass().getCanonicalName());
throw jpe;
}
+ try {
CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
-
// When running in e.g. Wildfly, the Gateway exists as two separate WARs - the API and the
// runtime Gateway itself. They both create a registry and thus they both try to initialize
// the ES index if it doesn't exist. A race condition could result in both WARs trying to
@@ -181,6 +183,17 @@ public abstract class AbstractClientFactory {
logger.info("ES index created: " + indexToCreate);
logger.debug("Creating an index => " + mapping);
}
+ } catch (ElasticsearchStatusException esse) {
+ // See: https://github.com/elastic/elasticsearch/issues/19862
+ if (esse.status() == RestStatus.BAD_REQUEST && (esse.getMessage().contains("index_already_exists_exception") || esse.getMessage().contains("resource_already_exists_exception"))) {
+ logger.warn("Exception occurred during attempt to create index. "
+ + "Elasticsearch does not offer create-if-not-exists for indexes, so this exception could be a race condition "
+ + "as another node already initialised the index: {0}", esse);
+ // Fall through.
+ } else {
+ throw esse;
+ }
+ }
}
}
