Tips on inputting information into a json file once you have completed crawling with casperjs?

Question

Tips on inputting information into a json file once you have completed crawling with casperjs?

I have written code that extracts a large number of words (innerHTML) from various webpages.

and I am interested in directly inserting this data into a json file..

Below is the JavaScript code snippet...

var words = [];
var casper = require('casper').create();

function getWords() {
    var words = document.querySelectorAll('td.subject a');
    return Array.prototype.map.call(words, function(e) {
        return e.innerHTML;
    });
}

casper.start('http://www.todayhumor.co.kr/board/list.php?table=bestofbest', function() {                
    words = this.evaluate(getWords);
});

for (var i=2; i <=5; i++) {
    casper.thenOpen('http://www.todayhumor.co.kr/board/list.php?table=bestofbest&page='+i, function() {              
        words = words.concat(this.evaluate(getWords));
    });
}

casper.run(function() {
    // Display results
    this.echo(words.length + ' links found:').exit();
    this.echo(words.join('\n')).exit();
});

Additionally,

I execute this code through terminal using the following command!

username@wow:~/workspace/app/assets/javascripts $ casperjs application.js

The result could be something like (example)

150 words found:
apple
banana
melon
kiwi
citrus
watermelon
passionfruit
mango
orange
...

Therefore, I aim to insert this data into the "word" section of my json file (as shown in the sample json code below)

Moreover, other columns ("type": "fruit" and "spell":) should be automatically included

{ "my_initial_words": [
    {
    "type": "fruit",
    "word": "apple",
    "spell": "ap"
    },
    {
    "type": "fruit",
    "word": "banana",
    "spell": "ba"
    },
    {
    "type": "fruit",
    "word": "melon",
    "spell": "me"
    }   
]
}
----------------------------------------------------------------------------

Thank you for providing additional information! However, I need guidance on where to incorporate this code

Can you please clarify which part of the code you provided executes the process of "Saving the results to JSON file?" as I intend to read the json file (makeyourap.json) in my seeds.rb file as follows

require 'json'
file = File.open(Rails.root.join('db','makeyourap.json'))
contents = file.read
json = ActiveSupport::JSON.decode(contents)["my_initial_words"]

javascript ruby-on-rails json casperjs

Answer 1

Answer №1

Is this the solution you were looking for?

function createTypeObject(name, category) {
  return {
    name: name,
    category: category,
    abbreviation: name.substr(0,2)
  };
}

var wordDescriptions = words.map(function (word) { 
   return createTypeObject(word, "fruit"); 
});

var finalOutput = {
  my_initial_words: wordDescriptions
};

var jsonData = JSON.stringify(finalOutput);
// For pretty printing, use JSON.stringify(finalOutput, null, "\t");

I trust that this clarifies things for you.

Answer 2

Is this the solution you were looking for?

function createTypeObject(name, category) {
  return {
    name: name,
    category: category,
    abbreviation: name.substr(0,2)
  };
}

var wordDescriptions = words.map(function (word) { 
   return createTypeObject(word, "fruit"); 
});

var finalOutput = {
  my_initial_words: wordDescriptions
};

var jsonData = JSON.stringify(finalOutput);
// For pretty printing, use JSON.stringify(finalOutput, null, "\t");

I trust that this clarifies things for you.

Answer 3

Answer №2

Using Casper to Write to a File

To create a file where you can both read and append content, you can follow this method:

var fs = require('fs');
var FILENAME = 'makeyourap.json';
function add_new_fruits(fruits) {
    var data;
    if ( fs.isFile(FILENAME) ) {
        data = fs.read(FILENAME);
    } else {
        data = JSON.stringify({'my_initial_words' : [] });
    }
    var json = JSON.parse(data);
    fruits.forEach(function(word) {
        json.my_initial_words.push({"type": "fruit",
                                    "name": word,
                                    "spell": word.slice(0,2)});
    });
    data = JSON.stringify(json, null, '\t');
    fs.write(FILENAME, data, "w");
}

Replace the older this.echo with this method. Simply call it like this:

casperjs application.js

This will either read the object from the file, or create it if it doesn't exist. Then, it will append each new object from the new fruits (even duplicates), and write it back to the FILENAME.

Previous Method: Creating Your Own Object

Create Object

Start by creating an object that only contains the parameter my_initial_words with values as specified.

You can use the following function:

function createFinal(wordArray) {
    var out = [];
    wordArray.forEach(function(word) {
        out.push({"type": "fruit", "name": word, "spell": word.slice(0,2)});
    });
    return out;
}

to generate the array. Then, create the object with:

var my_object = { "my_initial_words": createFinal(words) };

Converting to JSON

JavaScript has a built-in JSON-object. With a JavaScript object defined as

var my_object = { "my_initial_words": ...

as shown above, you can use

JSON.stringify(my_object)

to obtain the JSON representation for writing purposes.

Previous Method: Redirecting Output to File

Prior to the current approach, you may have used

this.echo(words.join('\n')).exit();

to display the list. Instead of using this.echo, consider replacing it with

var my_object = { "my_initial_words": createFinal(words) };
this.echo(JSON.stringify(my_object)).exit();

This will output to standard output. Delete the other this.echo line and redirect the output by running

casperjs application.js > makeyourap.json

If you need assistance with writing to a file in casperjs, refer to this guide.

Answer 4

Using Casper to Write to a File

To create a file where you can both read and append content, you can follow this method:

var fs = require('fs');
var FILENAME = 'makeyourap.json';
function add_new_fruits(fruits) {
    var data;
    if ( fs.isFile(FILENAME) ) {
        data = fs.read(FILENAME);
    } else {
        data = JSON.stringify({'my_initial_words' : [] });
    }
    var json = JSON.parse(data);
    fruits.forEach(function(word) {
        json.my_initial_words.push({"type": "fruit",
                                    "name": word,
                                    "spell": word.slice(0,2)});
    });
    data = JSON.stringify(json, null, '\t');
    fs.write(FILENAME, data, "w");
}

Replace the older this.echo with this method. Simply call it like this:

casperjs application.js

This will either read the object from the file, or create it if it doesn't exist. Then, it will append each new object from the new fruits (even duplicates), and write it back to the FILENAME.

Previous Method: Creating Your Own Object

Create Object

Start by creating an object that only contains the parameter my_initial_words with values as specified.

You can use the following function:

function createFinal(wordArray) {
    var out = [];
    wordArray.forEach(function(word) {
        out.push({"type": "fruit", "name": word, "spell": word.slice(0,2)});
    });
    return out;
}

to generate the array. Then, create the object with:

var my_object = { "my_initial_words": createFinal(words) };

Converting to JSON

JavaScript has a built-in JSON-object. With a JavaScript object defined as

var my_object = { "my_initial_words": ...

as shown above, you can use

JSON.stringify(my_object)

to obtain the JSON representation for writing purposes.

Previous Method: Redirecting Output to File

Prior to the current approach, you may have used

this.echo(words.join('\n')).exit();

to display the list. Instead of using this.echo, consider replacing it with

var my_object = { "my_initial_words": createFinal(words) };
this.echo(JSON.stringify(my_object)).exit();

This will output to standard output. Delete the other this.echo line and redirect the output by running

casperjs application.js > makeyourap.json

If you need assistance with writing to a file in casperjs, refer to this guide.

Tips on inputting information into a json file once you have completed crawling with casperjs?

Answer №1

Answer №2

Using Casper to Write to a File

Previous Method: Creating Your Own Object

Create Object

Converting to JSON

Previous Method: Redirecting Output to File

Similar questions

Volley's request queue cannot be utilized within a View Model

The process of deserializing JSON data in VB.Net

Sticky sidebar panel featuring a stationary content block

Create a separate child process in Node.js

Issue with PHP causing Jquery alert to trigger twice rather than once

Tips for getting rid of Next.js' default loading indicator

Is it possible to switch out all instances of "GET" methods with "POST" throughout the codebase?

Can you explain the purpose of the role attribute in XHTML? How is it commonly utilized?

Receiving inaccurate video duration when the input bar is modified

I need to show the JSON object value within an Android activity

I'm having trouble figuring out how to access response headers with HttpClient in Angular 5. Can anyone

How can a child class access this.props within a function that overrides a parent class's function?

Showing data retrieved from a JSON file on an Android device

Using " " to split a name into two lines is not being recognized

Issue with transmitting Razor form data to API controller using fetch or AJAX

JavaScript debugging causing system freeze

"The sliding function of the React Bootstrap carousel is malfunctioning as it goes blank just before transitioning

Retrieve a specific row from a table in Bootstrap

Importing classes in ECMAScript 6 does not work properly, causing issues when running scripts

Use of absolute positioning resulted in the disappearance of the element