Generate collections using an even quantity of items with the use of a loop

Question

Generate collections using an even quantity of items with the use of a loop

Currently, I am in the process of developing a website scraper using Javascript (Express) for my personal needs.

The main goal of this script is to extract basic text data from an external source and then convert it into JSON objects. However, I want to organize these objects in pairs inside an array, which is where I require some assistance from fellow developers like you.

At the moment, the output consists of generic JSON objects based on the number of extracted items (typically ranging from 8-16):

{
  name: "John Doe",
  email: "<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="afc5c0c7c181cbc0caefc5c0c7c181ccc0c2">[email protected]</a>,
  status: "active"
},

{
  name: "Jane Doe",
  email: "<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="85efe4ebe0abe1eae0c5efe4ebe0abe6eae8">[email protected]</a>,
  status: "inactive"
},


{
  name: "Johnny Walker",
  email: "<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="69030601070710471e0805020c1b291e0805020c1b470a0604">[email protected]</a>",
  status: "active"
},

{
  name: "Jimmy Glenfiddich",
  email: "<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="e48e8d89899dca8388818a828d80808d878ca48388818a80828d80808d878cca878b89">[email protected]</a>
  status: "active"
}

The desired outcome should resemble the following structure:

{

"pair-number": 1,

"pair:" [

  {
    name: "John Doe",
    email: "<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="4a20252224642e252f0a2025222464292527">[email protected]</a>,
    status: "active"
  },

  {
    name: "Jane Doe",
    email: "<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="c9a3a8a7ace7ada6ac89a3a8a7ace7aaa6a4">[email protected]</a>,
    status: "inactive"
  },
]
},

{

"pair-number": 2,

"pair:" [
  {
    name: "Johnny Walker",
    email: "<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="355f5a5d5b5b4c1b4254595e5047754254595e50471b565a58">[email protected]</a>",
    status: "active"
  },

  {
    name: "Jimmy Glenfiddich",
    email: "<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="462c2f2b2b3f68212a2328202f22222f252e06212a232822202f22222f252e6825292b">[email protected]</a>
    status: "active"
  }
]
}

This is how my `server.js` file looks like:

var express = require('express');
var request = require('request');
var cheerio = require('cheerio');
var app     = express();

var url = 'http://testurl.com;
var name, email, status;

app.get('/scrape', function(req, res) {

    request(url, function (error, response, html) {

        if (!error && response.statusCode == 200) {

            var $ = cheerio.load(html);

            data = {"name": name, "email": email, "status": status };       

            $('.scrape-class').filter(function() {

                var that = $(this);

                name = that.find('h5').text();
                email = that.find('.email').text(); 
                status = that.find('dl').children().first().text();

                data.name = name;
                data.email = email;
                data.status = status;

                console.log(data);
            });
        }
    });
});

app.listen(80, function () {
    console.log('Example app listening on port 80!')
})

javascript json express web-scraping

Answer 1

Answer №1

Utilize the modulo operator (%) to determine if an iteration is odd or even based on the remainder of a division operation.

Consider this basic concept:

var arr = [1,2,3,4,5,6];
var results = [];
for (var ii = 0, nn = arr.length; ii < nn; ii++)
{
   if (ii % 2 == 0) //0 % 2 = 0; 1 % 2 = 1; 2 % 2 = 0; 3 % 2 = 1;
   {
      //Even iteration: add the current value to a new array in the results array
      results.push([arr[ii]]);
   }
   else
   {
      //Odd iteration: add the current value to the last array in the results array
      results[results.length - 1].push(arr[ii]);
   }
}

//final results should be: [[1,2],[3,4],[5,6]]

In your scenario, maintain an external iteration count outside the filter function for it to work accordingly.

I made changes to your code and included a new results array:

var express = require('express');
var request = require('request');
var cheerio = require('cheerio');
var app     = express();

var url = 'http://testurl.com';
var name, email, status;

app.get('/scrape', function(req, res) {

    request(url, function (error, response, html) {

        if (!error && response.statusCode == 200) {

            var $ = cheerio.load(html);

            var results = []; //store final results here

            var data = {"name": name, "email": email, "status": status };

            var ii = 0;

            $('.scrape-class').filter(function() {

                var that = $(this);

                name = that.find('h5').text();
                email = that.find('.email').text(); 
                status = that.find('dl').children().first().text();

                data.name = name;
                data.email = email;
                data.status = status;

                console.log(data);

                if (ii % 2 == 0) { //0 % 2 = 0; 1 % 2 = 1; 2 % 2 = 0; 3 % 2 = 1;
                    //occurs on iterations: 0, 2, 4, etc
                    results.push({"pair-number": results.length + 1, "pair": [data]});
                } else {
                    //occurs on iterations: 1, 3, 5, etc
                    results[results.length - 1]["pair"].push(data);
                }

                ii++;
            });

            console.log(results); //display collected data after crawling
        }
    });
});

app.listen(80, function () {
    console.log('Example app listening on port 80!')
})

Answer 2

Utilize the modulo operator (%) to determine if an iteration is odd or even based on the remainder of a division operation.

Consider this basic concept:

var arr = [1,2,3,4,5,6];
var results = [];
for (var ii = 0, nn = arr.length; ii < nn; ii++)
{
   if (ii % 2 == 0) //0 % 2 = 0; 1 % 2 = 1; 2 % 2 = 0; 3 % 2 = 1;
   {
      //Even iteration: add the current value to a new array in the results array
      results.push([arr[ii]]);
   }
   else
   {
      //Odd iteration: add the current value to the last array in the results array
      results[results.length - 1].push(arr[ii]);
   }
}

//final results should be: [[1,2],[3,4],[5,6]]

In your scenario, maintain an external iteration count outside the filter function for it to work accordingly.

I made changes to your code and included a new results array:

var express = require('express');
var request = require('request');
var cheerio = require('cheerio');
var app     = express();

var url = 'http://testurl.com';
var name, email, status;

app.get('/scrape', function(req, res) {

    request(url, function (error, response, html) {

        if (!error && response.statusCode == 200) {

            var $ = cheerio.load(html);

            var results = []; //store final results here

            var data = {"name": name, "email": email, "status": status };

            var ii = 0;

            $('.scrape-class').filter(function() {

                var that = $(this);

                name = that.find('h5').text();
                email = that.find('.email').text(); 
                status = that.find('dl').children().first().text();

                data.name = name;
                data.email = email;
                data.status = status;

                console.log(data);

                if (ii % 2 == 0) { //0 % 2 = 0; 1 % 2 = 1; 2 % 2 = 0; 3 % 2 = 1;
                    //occurs on iterations: 0, 2, 4, etc
                    results.push({"pair-number": results.length + 1, "pair": [data]});
                } else {
                    //occurs on iterations: 1, 3, 5, etc
                    results[results.length - 1]["pair"].push(data);
                }

                ii++;
            });

            console.log(results); //display collected data after crawling
        }
    });
});

app.listen(80, function () {
    console.log('Example app listening on port 80!')
})

Generate collections using an even quantity of items with the use of a loop

Answer №1

Similar questions

Troubleshooting Error Message in Reactjs + Express Twitter API: "Authentication Failed"

What is the method for initiating a loop from index 1 rather than index 0 in an *ngFor loop within Angular?

Dealing with the "error" parameter in JSON in Angular and Ionic

Assigning the click action to a newly added div

The animatedModal.js dialog box is causing the link to be unclickable, despite having a z-index of -9999

When using Rails 6 with Turbolinks 5, the turbolinks:load event may trigger multiple times

"Exploring the process of utilizing React hooks to interact with a MongoDB Express server through POST and GET requests

The React state that is declared inside the render method cannot be accessed within the

Utilizing schema in JSON with Spark technology

Modify the JSON file stored on the local server

Solving yarn conflicts when managing multiple versions of a package

"Exploring the process of looping through a JSON object following an asynchronous retrieval of JSON data using

How can I connect a React component to a static HTML page?

Is it possible to retrieve a local variable from a JavaScript function and utilize it outside of its

AJAX: JSON response is not valid

Error message: npm command not recognized while running commands within an Electron application

Docker - Issue with locating module in docker container log

How can I retrieve a ref from a child component in Vue.js?

Transferring ajax data to a variable on a Node.js server

Steps for showcasing an image while performing in-memory processing with AngularJS