As a beginner in the world of web scraping and using Selenium, I am facing a challenge with a page that requires a JavaScript script on a button to navigate to the next page. I came across a snippet of code (Click a Button in Scrapy) on Stack Overflow, but unfortunately, I have been unable to get it to work.
from selenium import webdriver
import scrapy
class chSpider(scrapy.Spider):
name = 'spidypy'
allowed_domains = ['117.145.177.252']
start_urls = ['http://117.145.177.252/login.do?method=enterPdamccx']
def __init__(self):
self.driver = webdriver.Firefox()
def parse(self,response):
self.driver.get('http://117.145.177.252/login.do?method=enterPdamccx')
while True:
try:
next = self.driver.find_element_by_xpath('/html/body/form/div[3]/div/div/a')
url = 'http://117.145.177.252/login.do?method=enterPdamccx'
yield Request(url,callback=self.parse2)
next.click()
except:
break
self.driver.close()
def parse2(self,response):
print('you are here!')
I am encountering the following error message multiple times:
selenium.common.exceptions.WebDriverException: Message: connection refused