Googleの「関連する質問」セクションをスクレイピングする方法

mkdir people-also-ask-scraper
cd people-also-ask-scraper
python -m venv env

./env/bin/activate

env/Scripts/activate

pip install selenium

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options

# to control a Chrome window in headless mode
options = Options()
options.add_argument("--headless") # comment it while developing

# initialize a web driver instance with the
# specified options
driver = webdriver.Chrome(
    service=Service(),
    options=options
)

driver.get("https://google.com/")

driver.quit()

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options

# to control a Chrome window in headless mode
options = Options()
options.add_argument("--headless") # comment it while developing

# initialize a web driver instance with the
# specified options
driver = webdriver.Chrome(
    service=Service(),
    options=options
)

# connect to the Google home page
driver.get("https://google.com/")

# scraping logic...

# close the browser and free up the resources
driver.quit()

cookie_dialog = driver.find_element(By.CSS_SELECTOR, "[role='dialog']")

from selenium.webdriver.common.by import By

accept_button = cookie_dialog.find_element(By.XPATH, "//button[contains(., 'Accept')]")

try:
    # select the dialog and accept the cookie policy
    cookie_dialog = driver.find_element(By.CSS_SELECTOR, "[role='dialog']")
    accept_button = cookie_dialog.find_element(By.XPATH, "//button[contains(., 'Accept')]")
    if accept_button is not None:
        accept_button.click()
except NoSuchElementException:
    print("Cookie dialog not present")

from selenium.common import NoSuchElementException

search_form = driver.find_element(By.CSS_SELECTOR, "form[action='/search']")

from selenium.webdriver.common.by import By

search_textarea = search_form.find_element(By.CSS_SELECTOR, "textarea[aria-label='Search']")
search_query = "Bright Data"
search_textarea.send_keys(search_query)

search_form.submit()

people_also_ask_div = WebDriverWait(driver, 5).until(
    EC.presence_of_element_located((
        By.XPATH, "//div[@jscontroller and @jsname and @jsaction][.//div[@role='heading' and contains(., 'People also ask')]]"
    ))
)

from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

people_also_ask_questions = []

people_also_ask_inner_div = people_also_ask_div.find_element(By.CSS_SELECTOR, "[data-sgrd='true']")
people_also_ask_inner_div_children = people_also_ask_inner_div.find_elements(By.XPATH, "./*")
for child in people_also_ask_inner_div_children:
    # if the current element is a question dropdown
    if child.get_attribute("jsname") is not None and child.get_attribute("class") == '':
        # scraping logic...

child.click()

question_title_element = child.find_element(By.CSS_SELECTOR, "[aria-expanded='true'] span")
question_title = question_title_element.text

question_description_element = child.find_element(By.CSS_SELECTOR, "[data-attrid='wa:/description'] span[lang]")
question_description = question_description_element.text

try:
    question_image_element = child.find_element(By.CSS_SELECTOR, "img[data-ilt]")
    question_image = question_image_element.get_attribute("src")
except NoSuchElementException:
    question_image = None

from selenium.common import NoSuchElementException

question_source_element = child.find_element(By.XPATH, ".//h3/ancestor::a")
question_source = question_source_element.get_attribute("href")

people_also_ask_question = {
  "title": question_title,
  "description": question_description,
  "image": question_image,
  "source": question_source
}
people_also_ask_questions.append(people_also_ask_question)

[{'title': 'Is Bright Data legitimate?', 'description': 'Fast Residential Proxies from Bright Data is the industry standard for residential proxy networks. This network allows users to circumvent restrictions and bans by targeting any city, country, carrier, or ASN. It is reliable because it has over 72 million IP addresses obtained legally and an uptime of 99.99%.', 'image': 'https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSU5S3mnWcZeQPc2KOCp55dz1zrSX4I2WvV_vJxmvf9&s', 'source': 'https://www.linkedin.com/pulse/bright-data-review-legit-scam-everything-you-need-know-bloggrand-tiakc#:~:text=Fast%20Residential%20Proxies%20from%20Bright,and%20an%20uptime%20of%2099.99%25.'}, {'title': 'What is Bright Data used for?', 'description': "Bright Data is the world's #1 web data platform, supporting the public data needs of over 22,000 organizations in nearly every industry. Using our solutions, organizations research, monitor, and analyze web data to make better decisions.", 'image': None, 'source': "https://brightdata.com/about#:~:text=Bright%20Data%20is%20the%20world's,data%20to%20make%20better%20decisions."}, {'title': 'Is Bright Data legal?', 'description': "Bright Data's platform, technology, and network (collectively, “Services”) are meant for legitimate and legal purposes only and are subject to the Bright Data Master Service Agreement.", 'image': None, 'source': "https://brightdata.com/acceptable-use-policy#:~:text=Bright%20Data's%20platform%2C%20technology%2C%20and,Bright%20Data%20Master%20Service%20Agreement."}, {'title': 'Is Bright Data free?', 'description': 'Bright Data offers four free proxy solutions to meet various needs: Anonymous Proxies: These top-performing anonymous proxies let you access websites anonymously, routing traffic through a vast Residential IP Network of over 72 million IPs, concealing your true location.', 'image': None, 'source': 'https://brightdata.com/solutions/free-proxies#:~:text=Bright%20Data%20offers%20four%20free,IPs%2C%20concealing%20your%20true%20location.'}]

import csv

csv_file = "people_also_ask.csv"
header = ["title", "description", "image", "source"]
with open(csv_file, "w", newline="", encoding="utf-8") as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=header)
    writer.writeheader()
    writer.writerows(people_also_ask_questions)

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.common import NoSuchElementException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import csv

# to control a Chrome window in headless mode
options = Options()
options.add_argument("--headless") # comment it while developing

# initialize a web driver instance with the
# specified options
driver = webdriver.Chrome(
    service=Service(),
    options=options
)

# connect to the Google home page
driver.get("https://google.com/")

# deal with the optional Google cookie GDPR dialog
try:
    # select the dialog and accept the cookie policy
    cookie_dialog = driver.find_element(By.CSS_SELECTOR, "[role='dialog']")
    accept_button = cookie_dialog.find_element(By.XPATH, "//button[contains(., 'Accept')]")
    if accept_button is not None:
        accept_button.click()
except NoSuchElementException:
    print("Cookie dialog not present")

# select the search form
search_form = driver.find_element(By.CSS_SELECTOR, "form[action='/search']")

# select the textarea and fill it out
search_textarea = search_form.find_element(By.CSS_SELECTOR, "textarea[aria-label='Search']")
search_query = "Bright Data"
search_textarea.send_keys(search_query)

# submit the form to perform a Google search
search_form.submit()

# wait up to 5 seconds for the "People also ask" section
# to be on the page after page change
people_also_ask_div = WebDriverWait(driver, 5).until(
    EC.presence_of_element_located((
        By.XPATH, "//div[@jscontroller and @jsname and @jsaction][.//div[@role='heading' and contains(., 'People also ask')]]"
    ))
)

# where to store the scraped data
people_also_ask_questions = []

# select the question dropdowns and iterate over them
people_also_ask_inner_div = people_also_ask_div.find_element(By.CSS_SELECTOR, "[data-sgrd='true']")
people_also_ask_inner_div_children = people_also_ask_inner_div.find_elements(By.XPATH, "./*")
for child in people_also_ask_inner_div_children:
    # if the current element is a question dropdown
    if child.get_attribute("jsname") is not None and child.get_attribute("class") == '':
        # expand the element
        child.click()

        # scraping logic
        question_title_element = child.find_element(By.CSS_SELECTOR, "[aria-expanded='true'] span")
        question_title = question_title_element.text

        question_description_element = child.find_element(By.CSS_SELECTOR, "[data-attrid='wa:/description'] span[lang]")
        question_description = question_description_element.text

        try:
            question_image_element = child.find_element(By.CSS_SELECTOR, "img[data-ilt]")
            question_image = question_image_element.get_attribute("src")
        except NoSuchElementException:
            question_image = None

        question_source_element = child.find_element(By.XPATH, ".//h3/ancestor::a")
        question_source = question_source_element.get_attribute("href")

        # populate the array with the scraped data
        people_also_ask_question = {
          "title": question_title,
          "description": question_description,
          "image": question_image,
          "source": question_source
        }
        people_also_ask_questions.append(people_also_ask_question)

# export the scraped data to a CSV file
csv_file = "people_also_ask.csv"
header = ["title", "description", "image", "source"]
with open(csv_file, "w", newline="", encoding="utf-8") as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=header)
    writer.writeheader()
    writer.writerows(people_also_ask_questions)

# close the browser and free up the resources
driver.quit()

python scraper.py

python3 scraper.py