Deduplicate urls parsed to reduce crawl requests
added couple more URLs to fetch JDK package and seems to be causing some infrastructure as discussed in
This patch workaround the issue by removing the duplicated URLs and reduce the crawl request.
Closes
@@ -46,14 +46,19 @@ function test_verify_urls() {
# Find url-shaped lines, skipping jekyll-tree (which isn't a valid URL), and
# skipping comments.
invalid_urls=()
+ checked_urls=()
for file in "${WORKSPACE_FILES[@]}"; do
for url in $(grep -E '"https://|http://' "${file}" | \
sed -e '/jekyll-tree/d' -e '/^#/d' -r -e 's#^.*"(https?://[^"]+)".*$#\1#g' | \
sort -u); do
+ # add only unique url to the array
+ if [[ ${#checked_urls[@]} == 0 ]] || [[ ! " ${checked_urls[@]} " =~ " ${url} " ]]; then
+ checked_urls+=("${url}")
echo "Checking ${url} ..."
if ! curl --head --silent --show-error --fail --output /dev/null --retry 3 "${url}"; then
invalid_urls+=("${url}")
fi
+ fi
done
done
