Add logging in segment upload logic to make refresh scenarios traceable in logs
@@ -407,6 +407,9 @@ public class PinotSegmentUploadRestletResource {
TableConfig offlineTableConfig =
ZKMetadataProvider.getOfflineTableConfig(_pinotHelixResourceManager.getPropertyStore(), offlineTableName);
+ LOGGER.info("Starting upload for segment {}, tempTarredSegmentFile {}", segmentName,
+ tempTarredSegmentFile.getAbsolutePath());
+
if (offlineTableConfig == null) {
throw new ControllerApplicationException(LOGGER, "Failed to find table config for table: " + offlineTableName,
Response.Status.NOT_FOUND);
@@ -438,6 +441,7 @@ public class PinotSegmentUploadRestletResource {
// Brand new segment, not refresh, directly add the segment
if (znRecord == null) {
+ LOGGER.info("Adding new segment: {}", segmentName);
if (downloadUrl == null) {
downloadUrl = moveSegmentToPermanentDirectory(provider, rawTableName, segmentName, tempTarredSegmentFile);
}
@@ -445,7 +449,8 @@ public class PinotSegmentUploadRestletResource {
return;
}
- // Segment already exists, refresh if necessary
+ LOGGER.info("Segment {} already exists, refreshing if necessary", segmentName);
+
OfflineSegmentZKMetadata existingSegmentZKMetadata = new OfflineSegmentZKMetadata(znRecord);
long existingCrc = existingSegmentZKMetadata.getCrc();
@@ -518,13 +523,16 @@ public class PinotSegmentUploadRestletResource {
// Update ZK metadata and refresh the segment if necessary
long newCrc = Long.valueOf(segmentMetadata.getCrc());
if (newCrc == existingCrc) {
- // New segment is the same as the existing one, only update ZK metadata without refresh the segment
+ LOGGER.info("New segment crc {} is same as existing segment crc {} for segment {}. Updating ZK metadata without refreshing the segment {}",
+ newCrc, existingCrc, segmentName);
if (!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata)) {
throw new RuntimeException(
"Failed to update ZK metadata for segment: " + segmentName + " of table: " + offlineTableName);
}
} else {
// New segment is different with the existing one, update ZK metadata and refresh the segment
+ LOGGER.info("New segment crc {} is different than the existing segment crc {}. Updating ZK metadata and refreshing segment {}",
+ newCrc, existingCrc, segmentName);
if (downloadUrl == null) {
downloadUrl = moveSegmentToPermanentDirectory(provider, rawTableName, segmentName, tempTarredSegmentFile);
}
@@ -544,6 +552,8 @@ public class PinotSegmentUploadRestletResource {
File tarredSegmentFile = new File(new File(provider.getBaseDataDir(), tableName), segmentName);
FileUtils.deleteQuietly(tarredSegmentFile);
FileUtils.moveFile(tempTarredSegmentFile, tarredSegmentFile);
+ LOGGER.info("Moved segment {} from temp location {} to {}", segmentName, tempTarredSegmentFile.getAbsolutePath(),
+ tarredSegmentFile.getAbsolutePath());
return ControllerConf.constructDownloadUrl(tableName, segmentName, provider.getVip());
}
