Is there a way to extract data from a website once it has been fully loaded?
I have a process that iterates in a loop from 1 to 200, and I need to capture the results in the HTML once it reaches 200.
1. Can this be done? I am familiar with cheerio but unsure how to handle real-time data extraction after the process finishes.
2. How can I bypass CORS restrictions when making HTTP GET requests using axios?
I'm unclear on how to utilize a proxy in vue.config.js. The instructions I found were not comprehensive enough for my understanding.
Below is an example of my code (with some data altered for privacy reasons):
<div class="hello">
<h1>{{ msg }}</h1>
<ul>
<li v-for="(message, index) in messages" :key="index">
<b>{{ messages.ip }} [{{ message.type }}]:</b>
{{ message.blocked }}
</li>
</ul>
</div>
</template>
<script>
import axios from "axios";
import cheerio from "cheerio";
export default {
name: "ScrapIP",
props: {
msg: String,
messages: Array
},
methods: {
fetchUrl() {
for (let i = 0; i < 5; i++) {
const ip = "192.168.0." + i;
const url = "http://xxx/yyy.org/lookup/" + ip + ".html";
axios.get(url).then(response => {
const $ = cheerio.load(response.data);
setTimeout(() => {
if ($(".global_data_cnt_DNSBLBlacklistTest").text() == 243) {
this.messages.push({
ip: ip,
type: "Blacklist Test",
blocked: $(".global_data_cnt_DNSBLBlacklistTest").text()
});
}
}, 10000);
});
}
}
},
created() {
this.fetchUrl();
}
};
</script>