fix dataset/datasource lineage
@@ -24,6 +24,7 @@ import com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform
import com.thinkbiganalytics.metadata.api.catalog.DataSet;
import com.thinkbiganalytics.metadata.api.feed.Feed;
import com.thinkbiganalytics.metadata.rest.model.data.Datasource;
+import com.thinkbiganalytics.metadata.rest.model.data.DerivedDatasource;
import com.thinkbiganalytics.metadata.rest.model.feed.FeedDestination;
import com.thinkbiganalytics.metadata.rest.model.feed.FeedSource;
@@ -134,12 +135,19 @@ public class FeedLineageBuilder {
List<? extends com.thinkbiganalytics.metadata.api.feed.FeedSource> sources = domainFeed.getSources();
Set<FeedSource> feedSources = new HashSet<FeedSource>();
if (sources != null) {
- sources.stream().forEach(feedSource -> {
+ boolean containsDerivedDatasource = false;
+ for (com.thinkbiganalytics.metadata.api.feed.FeedSource feedSource : sources) {
FeedSource src = new FeedSource();
feedSource.getDatasource().ifPresent(datasource -> src.setDatasource(buildDatasource(datasource)));
+ //only add datasets if the source is missing or if its not a derived datasource
+ if (src.getDatasource() != null && src.getDatasource() instanceof DerivedDatasource) {
+ containsDerivedDatasource = true;
+ }
+ if (!containsDerivedDatasource) {
feedSource.getDataSet().ifPresent(dataSet -> src.setDatasource(buildDatasource(dataSet)));
+ }
feedSources.add(src);
- });
+ }
}
feed.setSources(feedSources);
Set<FeedDestination> feedDestinations = new HashSet<FeedDestination>();
