Only create gcs directory if it does not exist
If gcs gets similar prefixes in create calls it will rate limit the requests and throw a rate limit exceeded error. Code 429.
@@ -192,6 +192,9 @@ public class GcsPinotFS extends PinotFS {
if (path.equals(DELIMITER)) {
return true;
}
+ if (isDirectory(uri)) {
+ return true;
+ }
Blob blob = getBucket(uri).create(normalizeToDirectoryPrefix(uri), new byte[0]);
return blob.exists();
} catch (Throwable t) {
