Asad's solution is accurate; the issue arises from sending responses in each iteration of the loop. Since the request is fulfilled after the first iteration, attempting to respond again in the second iteration triggers an error in Express.
The minimal problematic scenario looks like this:
app.get("/news", (req, res) => {
for (let i = 0; i < 2; i++) {
res.json({message: "this won't work"});
}
});
The resolution is as follows:
app.get("/news", (req, res) => {
for (let i = 0; i < 2; i++) {
// perform necessary actions within the loop
}
// send response once the loop completes
res.json({message: "this works"});
});
An executable example of the fixed code:
const app = require("express")();
const axios = require("axios");
const cheerio = require("cheerio");
app.get("/news", (req, res) => {
axios
.get("https://www.theguardian.com/environment/climate-crisis")
.then((response) => {
const html = response.data;
const $ = cheerio.load(html);
var loopCounter = 0;
var climate_A_Elements = $("a:contains(climate)", html).length;
var allFoundArticles = $('a:contains("climate")', html);
const articlesData = [];
allFoundArticles
.each(function () {
var title = $(this).text();
var url = $(this).attr("href");
articlesData.push({
title,
url,
});
loopCounter += 1;
console.log("Loops: " + loopCounter);
})
res.json(articlesData);
});
});
app.listen(3000);
Test it:
$ curl localhost:3000/news | jq 'length'
14
$ curl localhost:3000/news | jq '.[0].title'
"Australia news Australia???s banks likely to reduce lending to regions and sectors at risk of climate change impacts, regulator says "
A simplified version without redundant code:
const app = require("express")();
const axios = require("axios");
const cheerio = require("cheerio");
app.get("/news", (req, res) => {
const url = "https://www.theguardian.com/environment/climate-crisis";
axios.get(url).then(({data}) => {
const $ = cheerio.load(data);
const articles = [...$('a:contains("climate")')].map(e => ({
title: $(e).text().trim(),
url: $(e).attr("href"),
}));
res.json(articles);
})
.catch(err => {
res.status(500).json({message: err.message});
});
});
app.listen(3000);
Alternatively, using async/await:
app.get("/news", async (req, res) => {
try {
const url = "https://www.theguardian.com/environment/climate-crisis";
const {data} = await axios.get(url);
const $ = cheerio.load(data);
const articles = [...$('a:contains("climate")')].map(e => ({
title: $(e).text().trim(),
url: $(e).attr("href"),
}));
res.json(articles);
}
catch (err) {
res.status(500).json({message: err.message});
}
});