我想要抓取 google 搜索结果,设置全局代理可以抓取,但是在 urllib 中就会报错[Errno 10060]
代理绝对可用,谁遇到过这种事情
我的代码
name=urllib.quote(wd)
proxy ='127.0.0.1:8787'
opener = urllib2.build_opener( urllib2.ProxyHandler({'socks':proxy}) )
urllib2.install_opener( opener )
url='https://www.google.co.jp/search?hl=en&q=intitle:%s+site:%s'%(name,url)
# url='http://www.baidu.com/s?wd=intitle:%s+site:%s'%(name,url)
request=urllib2.Request(url)
user_agents = ['Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0', \
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0', \
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533+ \
(KHTML, like Gecko) Element Browser 5.0', \
'IBM WebExplorer /v0.94', 'Galaxy/1.0 [en] (Mac OS X 10.5.6; U; en)', \
'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)', \
'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14', \
'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) \
Version/6.0 Mobile/10A5355d Safari/8536.25', \
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/28.0.1468.0 Safari/537.36', \
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; TheWorld)']
index = random.randint(0, 9)
user_agent = user_agents[index]
request.add_header('User-Agent',user_agent)
try:
html=urllib2.urlopen(request,timeout=120)
except urllib2.URLError, e:
print(e)
return False
else:
text=html.read()