python自动获取代理列表并通过代理爬取网站

# -*- coding: utf8 -*-
'''

'''

import urllib2
import urllib
import cookielib
import hashlib
import re
import time
import json
from pip._vendor.distlib._backport.tarfile import TUREAD

class Spide:
def __init__(self,proxy_ip,proxy_type,proxy_port,use_proxy=False):
print 'using the proxy info :',proxy_ip
self.proxy = urllib2.ProxyHandler({proxy_type: proxy_ip+":"+proxy_port})
self.usercode = ""
self.userid = ""
self.cj = cookielib.LWPCookieJar();

self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj));
if use_proxy:
self.opener = urllib2.build_opener(self.proxy)
urllib2.install_opener(self.opener);

#获取代理列表
def get_proxy(self):
proxy_info_json = ""
#first get the proxy info from
try:
reqRequest_proxy = urllib2.Request('http://g*XiXmX**mXeXpXrXoXxXy.com/api/getProxy');
reqRequest_proxy.add_header('Accept','*/*');
reqRequest_proxy.add_header('Accept-Language','zh-CN,zh;q=0.8');
reqRequest_proxy.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36');
reqRequest_proxy.add_header('Content-Type','application/x-www-form-urlencoded');


proxy_info = urllib2.urlopen(reqRequest_proxy).read();
print proxy_info
proxy_info_json = json.loads(proxy_info)
return_str=proxy_info_json['protocol']+":"+proxy_info_json['ip']+proxy_info_json['port']
except Exception,e:
print 'proxy have problem'
#print proxy_info_json['protocol']
#print proxy_info_json['ip']
#print proxy_info_json['port']
return proxy_info_json

#print proxy_info
def chrome(self):
try:

reqRequest = urllib2.Request('http://www.503*er**ror.com');
reqRequest.add_header('Accept','*/*');
reqRequest.add_header('Accept-Language','zh-CN,zh;q=0.8');
reqRequest.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36');
reqRequest.add_header('Content-Type','application/x-www-form-urlencoded');
content = urllib2.urlopen(reqRequest).read();
except Exception,e:
print 'oppps'



print 'done'



if __name__ == "__main__":

for count in range(100):
print '################################:',count
print 'Geting the new proxy info:'
test = Spide(proxy_ip='test',proxy_type='http',proxy_port='3128',use_proxy=False)
proxy_list = test.get_proxy()
#print proxy_list

print 'start to chrome'
spide1 = Spide(proxy_ip=proxy_list['ip'],proxy_type=proxy_list['protocol'],proxy_port=proxy_list['port'],use_proxy=True)
spide1.chrome()
time.sleep(5)