Selenium Integration

This guide covers using WeProxies with Selenium for browser automation and web scraping.

Python + Selenium

Installation

pip install selenium webdriver-manager

Chrome with HTTP Proxy

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager

PROXY_USER = "wp_user123"
PROXY_PASS = "password"
PROXY_HOST = "proxy.weproxies.com"
PROXY_PORT = "1080"

# Chrome options
options = Options()
options.add_argument(f'--proxy-server=http://{PROXY_HOST}:{PROXY_PORT}')

# For authenticated proxy, use extension (see below)
driver = webdriver.Chrome(
    service=Service(ChromeDriverManager().install()),
    options=options
)

driver.get("https://api.ipify.org?format=json")
print(driver.page_source)
driver.quit()

Chrome with Authenticated Proxy (Extension Method)

Chrome doesn't natively support authenticated proxies. Use a custom extension:

import os
import zipfile
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager

def create_proxy_extension(proxy_host, proxy_port, proxy_user, proxy_pass):
    """Create a Chrome extension for authenticated proxy."""
    
    manifest_json = """
    {
        "version": "1.0.0",
        "manifest_version": 2,
        "name": "Proxy Auth Extension",
        "permissions": [
            "proxy",
            "tabs",
            "unlimitedStorage",
            "storage",
            "<all_urls>",
            "webRequest",
            "webRequestBlocking"
        ],
        "background": {
            "scripts": ["background.js"]
        }
    }
    """
    
    background_js = f"""
    var config = {{
        mode: "fixed_servers",
        rules: {{
            singleProxy: {{
                scheme: "http",
                host: "{proxy_host}",
                port: parseInt({proxy_port})
            }},
            bypassList: ["localhost"]
        }}
    }};
    
    chrome.proxy.settings.set({{value: config, scope: "regular"}}, function() {{}});
    
    function callbackFn(details) {{
        return {{
            authCredentials: {{
                username: "{proxy_user}",
                password: "{proxy_pass}"
            }}
        }};
    }}
    
    chrome.webRequest.onAuthRequired.addListener(
        callbackFn,
        {{urls: ["<all_urls>"]}},
        ['blocking']
    );
    """
    
    extension_path = 'proxy_auth_extension.zip'
    
    with zipfile.ZipFile(extension_path, 'w') as zp:
        zp.writestr("manifest.json", manifest_json)
        zp.writestr("background.js", background_js)
    
    return extension_path

# Create extension
extension = create_proxy_extension(
    "proxy.weproxies.com", 
    "1080", 
    "wp_user123-country-US",  # With country targeting
    "password"
)

# Setup Chrome
options = Options()
options.add_extension(extension)
# options.add_argument('--headless')  # Uncomment for headless mode

driver = webdriver.Chrome(
    service=Service(ChromeDriverManager().install()),
    options=options
)

try:
    driver.get("https://api.ipify.org?format=json")
    print(driver.page_source)
finally:
    driver.quit()
    os.remove(extension)  # Cleanup

Firefox with Proxy

Firefox supports authenticated proxies natively:

from selenium import webdriver
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.firefox.options import Options
from webdriver_manager.firefox import GeckoDriverManager

PROXY_USER = "wp_user123"
PROXY_PASS = "password"
PROXY_HOST = "proxy.weproxies.com"
PROXY_PORT = 1080

# Firefox profile with proxy
profile = webdriver.FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", PROXY_HOST)
profile.set_preference("network.proxy.http_port", PROXY_PORT)
profile.set_preference("network.proxy.ssl", PROXY_HOST)
profile.set_preference("network.proxy.ssl_port", PROXY_PORT)

options = Options()
options.profile = profile

driver = webdriver.Firefox(
    service=Service(GeckoDriverManager().install()),
    options=options
)

driver.get("https://api.ipify.org?format=json")
print(driver.page_source)
driver.quit()

Using Selenium Wire (Recommended)

Selenium Wire makes proxy authentication easy:

pip install selenium-wire webdriver-manager

from seleniumwire import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

PROXY_USER = "wp_user123-country-US"
PROXY_PASS = "password"
PROXY_HOST = "proxy.weproxies.com"
PROXY_PORT = "1080"

# Selenium Wire options
seleniumwire_options = {
    'proxy': {
        'http': f'http://{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}',
        'https': f'http://{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}',
    }
}

driver = webdriver.Chrome(
    service=Service(ChromeDriverManager().install()),
    seleniumwire_options=seleniumwire_options
)

try:
    driver.get("https://api.ipify.org?format=json")
    print(driver.page_source)
finally:
    driver.quit()

Node.js + Selenium

Installation

npm install selenium-webdriver chromedriver

Basic Usage

const { Builder, Capabilities } = require('selenium-webdriver');
const chrome = require('selenium-webdriver/chrome');

const PROXY_USER = 'wp_user123';
const PROXY_PASS = 'password';
const PROXY_HOST = 'proxy.weproxies.com';
const PROXY_PORT = '1080';

async function runWithProxy() {
  const options = new chrome.Options();
  options.addArguments(`--proxy-server=http://${PROXY_HOST}:${PROXY_PORT}`);
  
  const driver = await new Builder()
    .forBrowser('chrome')
    .setChromeOptions(options)
    .build();
  
  try {
    await driver.get('https://api.ipify.org?format=json');
    const body = await driver.findElement({ tagName: 'body' }).getText();
    console.log(body);
  } finally {
    await driver.quit();
  }
}

runWithProxy();

Best Practices

1. Rotate IPs Between Sessions

import uuid
from seleniumwire import webdriver

def create_driver(country=None):
    session_id = uuid.uuid4().hex[:8]
    user = f"wp_user123-session-{session_id}"
    if country:
        user = f"wp_user123-country-{country}-session-{session_id}"
    
    options = {
        'proxy': {
            'http': f'http://{user}:password@proxy.weproxies.com:1080',
            'https': f'http://{user}:password@proxy.weproxies.com:1080',
        }
    }
    
    return webdriver.Chrome(seleniumwire_options=options)

# Each call gets a new IP
driver1 = create_driver("US")
driver2 = create_driver("GB")

2. Handle Page Load Timeouts

driver.set_page_load_timeout(60)  # 60 seconds

try:
    driver.get("https://example.com")
except TimeoutException:
    print("Page load timed out")
    driver.execute_script("window.stop();")

3. Use Headless Mode for Performance

options = Options()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')

4. Randomize User Agent

import random

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36...",
]

options = Options()
options.add_argument(f'user-agent={random.choice(USER_AGENTS)}')

Complete Example: Multi-Country Scraper

from seleniumwire import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
import time

def scrape_with_country(url, country):
    options = {
        'proxy': {
            'http': f'http://wp_user123-country-{country}:password@proxy.weproxies.com:1080',
            'https': f'http://wp_user123-country-{country}:password@proxy.weproxies.com:1080',
        }
    }
    
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--no-sandbox')
    
    driver = webdriver.Chrome(
        service=Service(ChromeDriverManager().install()),
        options=chrome_options,
        seleniumwire_options=options
    )
    
    try:
        driver.set_page_load_timeout(30)
        driver.get(url)
        time.sleep(2)  # Wait for dynamic content
        
        # Get page content
        body = driver.find_element(By.TAG_NAME, 'body').text
        return {"country": country, "content": body}
    except Exception as e:
        return {"country": country, "error": str(e)}
    finally:
        driver.quit()

# Scrape from multiple countries
countries = ["US", "GB", "DE"]
results = []

for country in countries:
    print(f"Scraping from {country}...")
    result = scrape_with_country("https://api.ipify.org?format=json", country)
    results.append(result)
    print(f"Result: {result}")