In the script provided, there is an array of URLs called links. The function gatherLinks() is designed to collect additional URLs from the sitemap.xml file related to the ones in the links array. Once the number of URLs in the links array reaches a certain limit determined by the variable limit, the function request() is executed for each URL in the links array. This function sends a request to the server, retrieves the response, and saves the image using the page.render() function.
An issue arises when running the script with PhantomJS 2.0.0 as many images end up missing content, indicating that PhantomJS may not be waiting for all the content to load. However, everything works fine when using PhantomJS 1.9.8. What could be causing this inconsistency?
var webpage = require('webpage');
var system = require('system');
var fs = require('fs');
var links = [];
links = [
"http://somesite.com",
"http://someothersite.com",
.
.
.
];
var index = 0, fail = 0, limit = 20;
finalTime = Date.now();
var gatherLinks = function(link){
var page = webpage.create();
link = link + "/sitemap.xml";
console.log("Fetching links from " + link);
page.open(link, function(status){
if(status != "success"){
console.log("Sitemap Request FAILED, status: " + status);
fail++;
return;
}
var content = page.content;
parser = new DOMParser();
xmlDoc = parser.parseFromString(content, 'text/xml');
var loc = xmlDoc.getElementsByTagName('loc');
for(var i = 0; i < loc.length; i++){
if(links.length < limit){
links[links.length] = loc[i].textContent;
} else{
console.log(links.length + " Links prepared. Starting requests.\n");
index = 0;
page.close();
request();
return;
}
}
if(index >= links.length){
index = 0;
console.log(links.length + " Links prepared\n\n");
page.close();
request();
return;
}
page.close();
gatherLinks(links[++index]);
});
};
var request = function(){
t = Date.now();
var page = webpage.create();
page.open(links[index], function(status) {
console.log('Loading link #' + (index + 1) + ': ' + links[index]);
console.log("Time taken: " + (Date.now() - t) + " msecs");
if(status != "success"){
console.log("Request FAILED, status: " + status);
fail++;
}
page.render("img_200_" + index + ".jpeg", {format: 'jpeg', quality: '100'});
if(index >= links.length-1){
console.log("\n\nAll links done, final time taken: " + (Date.now() - finalTime) + " msecs");
console.log("Requests sent: " + links.length + ", Failures: " + fail);
console.log("Success ratio: " + ((links.length - fail)/links.length)*100 + "%");
page.close();
phantom.exit();
}
index++;
page.close();
request();
});
}
gatherLinks(links[0]);