I'm struggling to retrieve data from multiple URLs and write it to a CSV file. The problem I'm facing is that the fetched data is not complete (I expect 10 items) and it's not in the correct order. Instead of getting 1, 2, 3 sequentially, I receive random numbers like 6, 10, 5, 1... Sometimes, I get six h3 values, sometimes five, and this discrepancy seems to happen randomly. Although my URL addresses are correct, using async await syntax hasn't resolved the issue. As a beginner, I've included my code below:
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const writeSteam = fs.createWriteStream('data.csv');
let data= '';
const numOfFetchData = 10;
const numbers = Array.from(Array(numOfFetchData + 1).keys());
async function getData() {
for await (const number of numbers) {
request('randomURL/' + (number+1), (err, res, html) => {
if(!err && res.statusCode == 200 && (number+1) <= numOfFetchData) {
const $ = cheerio.load(html);
const h3Tag = $("h3")[0].children[0].data;
data += (number + 1) + ' ' + h3Tag + '\n'
} else {
writeSteam.write(`${data}`);
}
});
};
};
getData();
What can I do to enhance my code?
Thank you and Regards!