use different crawl query (crawl all connection uris -> reduces our need to crawl into the connection itself, and therefore enhancing the connectionUri retrieval tremendously)
also make use of the localStorage to check if a given connuri is marked as closed, this will omit further crawling for this uri
@@ -10,6 +10,7 @@ import { ownerBaseUrl } from "config";
import urljoin from "url-join";
import { getRandomWonId } from "./won-utils.js";
+import { isConnUriClosed } from "./won-localstorage.js";
export const emptyDataset = Immutable.fromJS({
ownNeeds: {},
@@ -508,13 +509,20 @@ function fetchAllAccessibleAndRelevantData(
const allConnectionsPromise = allOwnNeedsPromise
.then(() =>
Promise.all(
- ownNeedUris.map(uri =>
+ ownNeedUris.map(needUri =>
won
- .getConnectionUrisOfNeed(uri, false)
+ .getConnectionUrisOfNeed(needUri, needUri, true)
.then(connectionUris =>
- urisToLookupMap(connectionUris, uri =>
- fetchConnectionAndDispatch(uri, curriedDispatch)
- )
+ urisToLookupMap(connectionUris, connUri => {
+ if (!isConnUriClosed(connUri)) {
+ fetchConnectionAndDispatch(connUri, curriedDispatch);
+ } else {
+ console.log(
+ connUri,
+ " closed, ommit further crawl for this connection"
+ );
+ }
+ })
)
)
)
