Question

python下 selenium与chrome结合进行网页爬取，怎么设置代理IP

Answer 1

#coding:utf-8
import sys,re,random,time,os
import socket
from socket import error as socket_error
import threading
import urllib2,cookielib
from bs4 import BeautifulSoup

from selenium import webdriver
from selenium.webdriver.common.proxy import *
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary

proxyFilePath = time.strftime("%Y%m%d")

def testSocket(ip, port):
'''
socket连接测试,用来检测proxy ip,port 是否可以正常连接
'''
print '正在测试socket连接...'
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.settimeout(10)
sock.connect((ip, int(port)))
#sock.send('meta')
sock.close()
print ip+':'+port+'--status:ok'
return 1
except socket_error as serr: # connection error
sock.close()
print ip+':'+port+'--status:error--Connection refused.'
return 0

def getDriver(httpProxy = '', type='Firefox'):
if type == 'Firefox':
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': httpProxy,
'ftpProxy': httpProxy,
'sslProxy': httpProxy,
'noProxy': '' # set this value as desired
})
firefox_profile = FirefoxProfile()
#firefox_profile.add_extension("firefox_extensions/adblock_plus-2.5.1-sm+tb+an+fx.xpi")
firefox_profile.add_extension("firefox_extensions/webdriver_element_locator-1.rev312-fx.xpi")
firefox_profile.set_preference("browser.download.folderList",2)
firefox_profile.set_preference("webdriver.load.strategy", "unstable")
#driver = webdriver.Firefox(firefox_profile = firefox_profile, proxy=proxy, firefox_binary=FirefoxBinary('/usr/bin/firefox'))
#driver = webdriver.Firefox(firefox_profile = firefox_profile, proxy=proxy, firefox_binary=FirefoxBinary("/cygdrive/c/Program\ Files\ (x86)/Mozilla\ Firefox/firefox.exe"))
driver = webdriver.Firefox(firefox_profile = firefox_profile, proxy=proxy)
elif type == 'PhantomJS': # PhantomJS
service_args = [
'--proxy='+httpProxy,
'--proxy-type=http',
]
webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.User-Agent'] = 'Mozilla/5.0 (X11; Windows x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36'
driver = webdriver.PhantomJS(executable_path='windows/phantomjs.exe', service_args=service_args)
else: # Chrome
chrome_options = webdriver.ChromeOptions()
#chrome_options.add_extension('firefox_extensions/adblockplus_1_7_4.crx')
chrome_options.add_argument('--proxy-server=%s' % httpProxy)
driver = webdriver.Chrome(executable_path='windows/chromedriver.exe', chrome_options=chrome_options)
return driver