PhantomJS: custom request headers

PhantomJS supports per-page custom headers via DesiredCapabilities.

from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

custom_headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
    "Connection": "keep-alive",
}

caps = DesiredCapabilities.PHANTOMJS.copy()
# Flatten headers into PhantomJS capability keys
caps.update({f"phantomjs.page.customHeaders.{k}": v for k, v in custom_headers.items()})

driver = webdriver.PhantomJS(desired_capabilities=caps)
try:
    driver.get("https://httpbin.org/get?show_env=1")
    driver.get_screenshot_as_file("phantom_headers.png")
finally:
    driver.quit()

Chrome: custom request headers

Basic UA and language can be adjusted with ChromeOptions arguments.

from selenium import webdriver

opts = webdriver.ChromeOptions()
opts.add_argument("lang=zh_CN.UTF-8")
opts.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36")

driver = webdriver.Chrome(options=opts)
try:
    driver.get("https://httpbin.org/headers")
    print(driver.page_source)
finally:
    driver.quit()

For arbitrary headers, use the Chrome DevTools Protocol via Selenium 4.

from selenium import webdriver

opts = webdriver.ChromeOptions()

driver = webdriver.Chrome(options=opts)
try:
    driver.execute_cdp_cmd("Network.enable", {})
    driver.execute_cdp_cmd(
        "Network.setExtraHTTPHeaders",
        {"headers": {"X-Requested-With": "XMLHttpRequest", "X-Demo": "1"}},
    )
    driver.execute_cdp_cmd(
        "Network.setUserAgentOverride",
        {"userAgent": "Mozilla/5.0 CustomUA Chrome/114.0 Safari/537.36"},
    )

    driver.get("https://httpbin.org/headers")
finally:
    driver.quit()

Chrome: manage cookies

Cookies can be cleared, added, and reused across tabs to the same domain.

from selenium import webdriver

driver = webdriver.Chrome()
try:
    driver.get("https://www.baidu.com/")

    # Remove existing cookies, then add a new one
    driver.delete_all_cookies()
    driver.add_cookie({"name": "ABC", "value": "DEF"})

    # Open a new tab on the same domain to reuse the cookie
    driver.execute_script('window.open("https://www.baidu.com", "_blank");')
    driver.switch_to.window(driver.window_handles[-1])

    # Verify cookies in the new tab
    print(driver.get_cookies())
finally:
    driver.quit()

Chrome: block images to reduce bandwidth

Disable image fetching via profile preferences.

from selenium import webdriver

opts = webdriver.ChromeOptions()
prefs = {"profile.managed_default_content_settings.images": 2}
opts.add_experimental_option("prefs", prefs)

driver = webdriver.Chrome(options=opts)
try:
    driver.get("https://image.baidu.com/")
finally:
    driver.quit()