"""

proxy getter

"""

@staticmethod

def freeProxy01():

"""

无忧代理 http://www.data5u.com/

几乎没有能用的

:return:

"""

url_list = [

'http://www.data5u.com/',

'http://www.data5u.com/free/gngn/index.shtml',

'http://www.data5u.com/free/gnpt/index.shtml'

]

key = 'ABCDEFGHIZ'

for url in url_list:

html_tree = WebRequest().get(url).tree

ul_list = html_tree.xpath('//ul[@class="l2"]')

for ul in ul_list:

try:

ip = ul.xpath('./span[1]/li/text()')[0]

classnames = ul.xpath('./span[2]/li/attribute::class')[0]

classname = classnames.split(' ')[1]

port_sum = 0

for c in classname:

port_sum *= 10

port_sum += key.index(c)

port = port_sum >> 3

yield '{}:{}'.format(ip, port)

except Exception as e:

print(e)

@staticmethod

def freeProxy02(count=20):

"""

代理66 http://www.66ip.cn/

:param count: 提取数量

:return:

"""

urls = [

"http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=",

"http://www.66ip.cn/nmtq.php?getnum={}&isp=0&anonymoustype=0&s"

"tart=&ports=&export=&ipaddress=&area=0&proxytype=2&api=66ip"

]

try:

import execjs

import requests

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',

'Accept': '*/*',

'Connection': 'keep-alive',

'Accept-Language': 'zh-CN,zh;q=0.8'}

session = requests.session()

src = session.get("http://www.66ip.cn/", headers=headers).text

src = src.split("</script>")[0] + '}'

src = src.replace("<script>", "function test() {")

src = src.replace("while(z++)try{eval(", ';var num=10;while(z++)try{var tmp=')

src = src.replace(");break}", ";num--;if(tmp.search('cookie') != -1 | num<0){return tmp}}")

ctx = execjs.compile(src)

src = ctx.call("test")

src = src[src.find("document.cookie="): src.find("};if((")]

src = src.replace("document.cookie=", "")

src = "function test() {var window={}; return %s }" % src

cookie = execjs.compile(src).call('test')

js_cookie = cookie.split(";")[0].split("=")[-1]

except Exception as e:

print(e)

return

for url in urls:

try:

html = session.get(url.format(count), cookies={"__jsl_clearance": js_cookie}, headers=headers).text

ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", html)

for ip in ips:

yield ip.strip()

except Exception as e:

print(e)

pass

@staticmethod

def freeProxy03(page_count=1):

"""

西刺代理 http://www.xicidaili.com

:return:

"""

url_list = [

'http://www.xicidaili.com/nn/', # 高匿

'http://www.xicidaili.com/nt/', # 透明

]

for each_url in url_list:

for i in range(1, page_count + 1):

page_url = each_url + str(i)

tree = WebRequest().get(page_url).tree

proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]')

for proxy in proxy_list:

try:

yield ':'.join(proxy.xpath('./td/text()')[0:2])

except Exception as e:

pass

@staticmethod

def freeProxy04():

"""

guobanjia http://www.goubanjia.com/

:return:

"""

url = "http://www.goubanjia.com/"

tree = WebRequest().get(url).tree

proxy_list = tree.xpath('//td[@class="ip"]')

# 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号

# 需要过滤掉<p style="display:none;">的内容

xpath_str = """.//*[not(contains(@style, 'display: none'))

and not(contains(@style, 'display:none'))

and not(contains(@class, 'port'))

]/text()

"""

for each_proxy in proxy_list:

try:

# :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port

ip_addr = ''.join(each_proxy.xpath(xpath_str))

# HTML中的port是随机数，真正的端口编码在class后面的字母中。

# 比如这个：

# <span class="port CFACE">9054</span>

# CFACE解码后对应的是3128。

port = 0

for _ in each_proxy.xpath(".//span[contains(@class, 'port')]"

"/attribute::class")[0]. \

replace("port ", ""):

port *= 10

port += (ord(_) - ord('A'))

port /= 8

yield '{}:{}'.format(ip_addr, int(port))

except Exception as e:

pass

@staticmethod

def freeProxy05():

"""

快代理 https://www.kuaidaili.com

"""

url_list = [

'https://www.kuaidaili.com/free/inha/',

'https://www.kuaidaili.com/free/intr/'

]

for url in url_list:

tree = WebRequest().get(url).tree

proxy_list = tree.xpath('.//table//tr')

sleep(1) # 必须sleep 不然第二条请求不到数据

for tr in proxy_list[1:]:

yield ':'.join(tr.xpath('./td/text()')[0:2])

@staticmethod

def freeProxy06():

"""

码农代理 https://proxy.coderbusy.com/

:return:

"""

urls = ['https://proxy.coderbusy.com/']

for url in urls:

tree = WebRequest().get(url).tree

proxy_list = tree.xpath('.//table//tr')

for tr in proxy_list[1:]:

yield ':'.join(tr.xpath('./td/text()')[0:2])

@staticmethod

def freeProxy07():

"""

云代理 http://www.ip3366.net/free/

:return:

"""

urls = ['http://www.ip3366.net/free/?stype=1',

"http://www.ip3366.net/free/?stype=2"]

for url in urls:

r = WebRequest().get(url, timeout=10)

proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)

for proxy in proxies:

yield ":".join(proxy)

@staticmethod

def freeProxy08():

"""

IP海 http://www.iphai.com/free/ng

:return:

"""

urls = [

'http://www.iphai.com/free/ng',

'http://www.iphai.com/free/np',

'http://www.iphai.com/free/wg',

'http://www.iphai.com/free/wp'

]

for url in urls:

r = WebRequest().get(url, timeout=10)

proxies = re.findall(r'<td>\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?</td>[\s\S]*?<td>\s*?(\d+)\s*?</td>',

r.text)

for proxy in proxies:

yield ":".join(proxy)

@staticmethod

def freeProxy09(page_count=1):

"""

http://ip.jiangxianli.com/?page=

免费代理库

:return:

"""

for i in range(1, page_count + 1):

url = 'http://ip.jiangxianli.com/?country=中国&?page={}'.format(i)

html_tree = WebRequest().get(url).tree

for index, tr in enumerate(html_tree.xpath("//table//tr")):

if index == 0:

continue

yield ":".join(tr.xpath("./td/text()")[0:2]).strip()

# @staticmethod

# def freeProxy10():

# """

# 墙外网站 cn-proxy

# :return:

# """

# urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']

# request = WebRequest()

# for url in urls:

# r = request.get(url, timeout=10)

# proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)

# for proxy in proxies:

# yield ':'.join(proxy)

# @staticmethod

# def freeProxy11():

# """

# https://proxy-list.org/english/index.php

# :return:

# """

# urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)]

# request = WebRequest()

# import base64

# for url in urls:

# r = request.get(url, timeout=10)

# proxies = re.findall(r"Proxy\('(.*?)'\)", r.text)

# for proxy in proxies:

# yield base64.b64decode(proxy).decode()

# @staticmethod

# def freeProxy12():

# urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1']

# request = WebRequest()

# for url in urls:

# r = request.get(url, timeout=10)

# proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)

# for proxy in proxies:

# yield ':'.join(proxy)

@staticmethod

def freeProxy13(max_page=2):

"""

http://www.qydaili.com/free/?action=china&page=1

齐云代理

:param max_page:

:return:

"""

base_url = 'http://www.qydaili.com/free/?action=china&page='

for page in range(1, max_page + 1):

url = base_url + str(page)

r = WebRequest().get(url, timeout=10)

proxies = re.findall(

r'<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td.*?>(\d+)</td>',

r.text)

for proxy in proxies:

yield ':'.join(proxy)

@staticmethod

def freeProxy14(max_page=2):

"""

http://www.89ip.cn/index.html

89免费代理

:param max_page:

:return:

"""

base_url = 'http://www.89ip.cn/index_{}.html'

for page in range(1, max_page + 1):

url = base_url.format(page)

r = WebRequest().get(url, timeout=10)

proxies = re.findall(

r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',

r.text)

for proxy in proxies:

yield ':'.join(proxy)

@staticmethod

def freeProxy15():

urls = ['http://www.xiladaili.com/putong/',

"http://www.xiladaili.com/gaoni/",

"http://www.xiladaili.com/http/",

"http://www.xiladaili.com/https/"]

for url in urls:

r = WebRequest().get(url, timeout=10)

ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", r.text)

for ip in ips: