I've successfully coded a script that scrapes data from the first page, but I encountered an issue when implementing a loop to click on a "load more" button to fetch additional data. After running the loop, the code doesn't export anything to CSV. Is there something wrong with my exporting code? Can someone help me figure out where I'm making a mistake?
const puppeteer = require('puppeteer');
const jsonexport = require('jsonexport');
(async () => {
const browser = await puppeteer.launch({ headless: false }); // default is true
const page = await browser.newPage();
await page.goto('https://www.Website.com/exercises/finder', {
waitUntil: 'domcontentloaded',
});
//load more CSS to be targeted
const LoadMoreButton =
'#js-ex-content > #js-ex-category-body > .ExCategory-results > .ExLoadMore > .bb-flat-btn';
do {
// clicking load more button and waiting 1sec
await page.click(LoadMoreButton);
await page.waitFor(1000);
const loadMore = true;
const rowsCounts = await page.$eval(
'.ExCategory-results > .ExResult-row',
(rows) => rows.length
);
//scraping the data
const exerciseNames = [];
for (let i = 2; i < rowsCounts + 1; i++) {
const exerciseName = await page.$eval(
`.ExCategory-results > .ExResult-row:nth-child(${i}) > .ExResult-cell > .ExHeading > a`,
(el) => el.innerText
);
exerciseNames.push(exerciseName);
}
console.log({exerciseNames});
} while (There are still exercises left);
const allData = [
{
exercise: exerciseNames,
},
];
// exporting data to CSV
const options = [exercise];
//json export error part
jsonexport(allData, options, function (err, csv) {
if (err) return console.error(err);
console.log(csv);
});
await browser.close();
})().catch((e) => {
console.error(e);
});
Edit: I have made changes to the exporting and writing to a CSV file portion of the code. Currently, only the exercises are being written with 3 headers visible in the CSV. I want to format it so that each row contains data for exercise name, equipment type, and muscle target group respectively.
Current export code:
const allData = [
{
exercise: exerciseNames,
muscleGroup: muscleTargets,
equipment: equipmentTypes,
},
];
var ws = fs.createWriteStream('test1.csv');
csv.write(allData, { headers: true, delimiter: ',' }).pipe(ws);
//json export error part
jsonexport(allData, function (err, csv) {
if (err) return console.error(err);
console.log(csv);
});
https://i.sstatic.net/YFD5P.png
Edit 2 This block shows my complete code. While it outputs pre-filled information from allData, new data is not being fetched or exported. I need assistance in fixing this issue.
const puppeteer = require('puppeteer');
const jsonexport = require('jsonexport');
const fs = require('fs');
(async () => {
const browser = await puppeteer.launch({ headless: false }); // default is true
const page = await browser.newPage();
await page.goto('https://www.website.com/exercises/finder', {
waitUntil: 'domcontentloaded',
});
const loadMore = true;
const rowsCounts = await page.$$eval(
'.ExCategory-results > .ExResult-row',
(rows) => rows.length
);
let allData = [];
for (let i = 2; i < rowsCounts + 1; i++) {
const exerciseName = await page.$eval(
`.ExCategory-results > .ExResult-row:nth-child(${i}) > .ExResult-cell > .ExHeading > a`,
(el) => el.innerText
);
const muscleGroupName = await page.$eval(
`.ExCategory-results > .ExResult-row:nth-child(${i}) > .ExResult-cell > .ExResult-muscleTargeted > a`,
(el) => el.innerHTML
);
const equipmentName = await page.$eval(
`.ExCategory-results > .ExResult-row:nth-child(${i}) > .ExResult-cell > .ExResult-equipmentType > a`,
(el) => el.innerHTML
);
let obj = {
exercise: exerciseName,
muscleGroup: muscleGroupName,
equipment: equipmentName,
};
allData.push(obj);
}
console.log(allData);
async function fn() {
const allData = [
{
exercise: 'Rickshaw Carry',
muscleGroup: 'Forearms',
equipment: 'Other',
},
{
exercise: 'Single-Leg Press',
muscleGroup: 'Quadriceps',
equipment: 'Machine',
},
{
exercise: 'Landmine twist',
muscleGroup: 'Forearms',
equipment: 'Other',
},
{
exercise: 'Weighted pull-up',
muscleGroup: 'Forearms',
equipment: 'Other',
},
];
jsonexport(allData, function (err, csv) {
if (err) return console.error(err);
console.log(csv);
fs.writeFileSync('output.csv', csv);
});
}
fn();
await browser.close();
})().catch((e) => {
console.error(e);
});