Spaces:
Running
Running
File size: 2,849 Bytes
9b9ea2f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
# Import necessary libraries
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
import time
"""
This code uses Selenium to scrape data from a webpage.
It initializes a Chrome webdriver and loads the webpage defined by the url variable.
It then clicks a modal pop-up that appears when the webpage is loaded.
The code then enters a loop to scrape data from each page of the webpage.
It locates the data tables and extracts the table rows.
It loops through each row of the table and extracts the name, region, and description from the row.
It appends these values to the dai_values list.
The code then finds the 'Next' button and checks if it is disabled.
If the button is disabled, the loop is broken.
If the button is not disabled, the code clicks the button, scrolls to it, and waits for 2 seconds before moving on to the next page.
Finally, the code quits the webdriver.
"""
# Define the URL of the webpage to be scraped
url = 'https://www.hdrn.ca/en/inventory/'
# Initialize a Chrome webdriver
driver = webdriver.Chrome()
driver.get(url)
# Define a wait time for the driver to locate web elements
wait = WebDriverWait(driver, 2)
# Create an empty list to store the scraped data
dai_values = []
# Click the modal pop-up that appears when the webpage is loaded
driver.find_element(By.ID, 'myModal').click()
# Loop through the webpage to scrape data from each page
while True:
# Locate the data tables and extract the table rows
data_tables_scroll = driver.find_elements(By.CLASS_NAME, 'dataTables_scrollBody')[-1]
table = data_tables_scroll.find_elements(By.TAG_NAME, 'tr')
# Loop through each row of the table
for row in table:
# Extract the values from each cell of the row
row_values = row.find_elements(By.TAG_NAME, 'td')
# If the row has less than 2 cells, skip to the next row
if len(row_values) < 2:
continue
# Extract the name, region, and description from the row and append to the dai_values list
name, region, description = row_values
dai_values.append({
'name': name.text,
'region': region.text,
'description': description.text
})
# Find the 'Next' button and check if it is disabled
next_button = driver.find_elements(By.ID, 'thelist_next')
if 'disabled' in next_button[0].get_attribute('class'):
# If the button is disabled, break out of the loop
break
else:
# If the button is not disabled, click it, scroll to it, and wait for 2 seconds before moving on to the next page
driver.click()
driver.execute_script("arguments[0].scrollIntoView();", next_button[0])
time.sleep(2)
next_button[0].click()
# Quit the webdriver
driver.quit()
|