bugfix
This commit is contained in:
parent
e5d72b67bf
commit
ec090a205e
@ -1,3 +1,4 @@
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
from aiohttp import FormData
|
||||
@ -9,13 +10,13 @@ from urllib.parse import urlparse
|
||||
from aiohttp_socks import ProxyType, ProxyConnector, ChainProxyConnector
|
||||
|
||||
def get_domain(url):
|
||||
# Prepend 'http://' if the URL lacks a scheme
|
||||
if not url.startswith(('http://', 'https://')):
|
||||
url = 'http://' + url
|
||||
parsed_url = urlparse(url)
|
||||
netloc = parsed_url.netloc
|
||||
domain = netloc.split(':')[0]
|
||||
return domain
|
||||
# Prepend 'http://' if the URL lacks a scheme
|
||||
if not url.startswith(('http://', 'https://')):
|
||||
url = 'http://' + url
|
||||
parsed_url = urlparse(url)
|
||||
netloc = parsed_url.netloc
|
||||
domain = netloc.split(':')[0]
|
||||
return domain
|
||||
|
||||
RESPONSE_BIN = 0
|
||||
RESPONSE_TEXT = 1
|
||||
@ -39,34 +40,33 @@ class HttpClient:
|
||||
self.coding = coding
|
||||
self.session = None
|
||||
self.cookies = {}
|
||||
self.socks5proxy = socks5proxy
|
||||
self.proxy_connector = None
|
||||
self.socks5_proxy_url = socks5_proxy_url
|
||||
self.blocked_domains = set()
|
||||
self.load_cache()
|
||||
self.proxy_connector = None
|
||||
self.socks5_proxy_url = socks5_proxy_url
|
||||
self.blocked_domains = set()
|
||||
self.load_cache()
|
||||
|
||||
def save_cache(self):
|
||||
home_dir = os.path.expanduser('~')
|
||||
cache_file = os.path.join(home_dir, '.proxytarget')
|
||||
with open(cache_file, 'w') as f:
|
||||
for d in self.blocked_domains:
|
||||
f.write(f'{d}\n')
|
||||
def save_cache(self):
|
||||
home_dir = os.path.expanduser('~')
|
||||
cache_file = os.path.join(home_dir, '.proxytarget')
|
||||
with open(cache_file, 'w') as f:
|
||||
for d in self.blocked_domains:
|
||||
f.write(f'{d}\n')
|
||||
|
||||
def load_cache(self):
|
||||
# 初始化缓存文件
|
||||
home_dir = os.path.expanduser('~')
|
||||
cache_file = os.path.join(home_dir, '.proxytarget')
|
||||
|
||||
try:
|
||||
with open(cache_file, 'r') as f:
|
||||
for line in f:
|
||||
domain = line.strip()
|
||||
if domain:
|
||||
self.blocked_domains.add(domain)
|
||||
except FileNotFoundError:
|
||||
# 创建空文件
|
||||
with open(cache_file, 'w') as f:
|
||||
pass
|
||||
def load_cache(self):
|
||||
# 初始化缓存文件
|
||||
home_dir = os.path.expanduser('~')
|
||||
cache_file = os.path.join(home_dir, '.proxytarget')
|
||||
|
||||
try:
|
||||
with open(cache_file, 'r') as f:
|
||||
for line in f:
|
||||
domain = line.strip()
|
||||
if domain:
|
||||
self.blocked_domains.add(domain)
|
||||
except FileNotFoundError:
|
||||
# 创建空文件
|
||||
with open(cache_file, 'w') as f:
|
||||
pass
|
||||
|
||||
async def close(self):
|
||||
if self.session:
|
||||
@ -107,88 +107,88 @@ class HttpClient:
|
||||
filtered = session.cookie_jar.filter_cookies(domain)
|
||||
return filtered
|
||||
|
||||
async def make_request(self, url, method,
|
||||
async def make_request(self, url, method='GET',
|
||||
response_type=RESPONSE_TEXT,
|
||||
params=None,
|
||||
data=None,
|
||||
jd=None,
|
||||
stream_func=None,
|
||||
headers=None,
|
||||
use_proxy=False,
|
||||
**kw):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
if params == {}:
|
||||
params = None
|
||||
if data == {}:
|
||||
data = None
|
||||
if jd == {}:
|
||||
jd = None
|
||||
proxy = None
|
||||
if use_proxy and self.socks5_proxy_url:
|
||||
proxy = self.socks5_proxy_url
|
||||
use_proxy=False
|
||||
):
|
||||
connector = None
|
||||
if use_proxy:
|
||||
connector = ProxyConnector.from_url(self.socks5_proxy_url)
|
||||
async with aiohttp.ClientSession(connector=connector) as session:
|
||||
if params == {}:
|
||||
params = None
|
||||
if data == {}:
|
||||
data = None
|
||||
if jd == {}:
|
||||
jd = None
|
||||
|
||||
if headers == {}:
|
||||
headers = None
|
||||
if headers == {}:
|
||||
headers = None
|
||||
|
||||
resp = await session.request(method, url,
|
||||
params=params,
|
||||
data=data,
|
||||
json=jd,
|
||||
proxy=proxy,
|
||||
headers=headers, **kw)
|
||||
if resp.status==200:
|
||||
return await self.handleResp(url, resp, response_type, stream_func=stream_func)
|
||||
msg = f'http error({resp.status}, {url=},{params=}, {data=}, {jd=}, {headers=}, {kw=})'
|
||||
exception(msg)
|
||||
raise HttpError(resp.status, msg)
|
||||
resp = await session.request(method, url,
|
||||
params=params,
|
||||
data=data,
|
||||
json=jd,
|
||||
headers=headers)
|
||||
if resp.status==200:
|
||||
return await self.handleResp(url, resp, response_type, stream_func=stream_func)
|
||||
msg = f'http error({resp.status}, {url=},{params=}, {data=}, {jd=}, {headers=}, {kw=})'
|
||||
exception(msg)
|
||||
raise HttpError(resp.status, msg)
|
||||
|
||||
async def request(self, url, method,
|
||||
response_type=RESPONSE_TEXT,
|
||||
params=None,
|
||||
data=None,
|
||||
jd=None,
|
||||
stream_func=None,
|
||||
headers=None,
|
||||
**kw
|
||||
):
|
||||
if self.socks5_proxy_url is None:
|
||||
resp = await self.make_request(url, method,
|
||||
response_type=response_type,
|
||||
params=params,
|
||||
data=data,
|
||||
jd=jd,
|
||||
proxy=False,
|
||||
stream_func=stream_func,
|
||||
headers=headers,
|
||||
**kw)
|
||||
return resp
|
||||
domain = get_domain(url)
|
||||
if domain not in self.blocked_domains:
|
||||
try:
|
||||
resp = await self.make_request(url, method,
|
||||
response_type=response_type,
|
||||
params=params,
|
||||
data=data,
|
||||
jd=jd,
|
||||
proxy=False,
|
||||
stream_func=stream_func,
|
||||
headers=headers,
|
||||
**kw)
|
||||
return resp
|
||||
except aiohttp.ClientError:
|
||||
if domain not in self.blocked_domains:
|
||||
self.save_cache()
|
||||
resp = await self.make_request(url, method,
|
||||
response_type=response_type,
|
||||
params=params,
|
||||
data=data,
|
||||
jd=jd,
|
||||
proxy=True,
|
||||
stream_func=stream_func,
|
||||
headers=headers,
|
||||
**kw)
|
||||
return resp
|
||||
|
||||
async def request(self, url, method='GET',
|
||||
response_type=RESPONSE_TEXT,
|
||||
params=None,
|
||||
data=None,
|
||||
jd=None,
|
||||
stream_func=None,
|
||||
headers=None,
|
||||
**kw
|
||||
):
|
||||
if self.socks5_proxy_url is None:
|
||||
resp = await self.make_request(url, method=method,
|
||||
response_type=response_type,
|
||||
params=params,
|
||||
data=data,
|
||||
jd=jd,
|
||||
use_proxy=False,
|
||||
stream_func=stream_func,
|
||||
headers=headers
|
||||
)
|
||||
return resp
|
||||
domain = get_domain(url)
|
||||
if domain not in self.blocked_domains:
|
||||
try:
|
||||
resp = await self.make_request(url, method=method,
|
||||
response_type=response_type,
|
||||
params=params,
|
||||
data=data,
|
||||
jd=jd,
|
||||
use_proxy=False,
|
||||
stream_func=stream_func,
|
||||
headers=headers
|
||||
)
|
||||
return resp
|
||||
except:
|
||||
if domain not in self.blocked_domains:
|
||||
self.blocked_domains.add(domain)
|
||||
self.save_cache()
|
||||
resp = await self.make_request(url, method=method,
|
||||
response_type=response_type,
|
||||
params=params,
|
||||
data=data,
|
||||
jd=jd,
|
||||
use_proxy=True,
|
||||
stream_func=stream_func,
|
||||
headers=headers
|
||||
)
|
||||
return resp
|
||||
|
||||
async def get(self,url,**kw):
|
||||
return self.request(url, 'GET', **kw)
|
||||
|
||||
@ -197,10 +197,10 @@ class HttpClient:
|
||||
session = self.getsession(url)
|
||||
|
||||
class JsonHttpAPI:
|
||||
def __init__(self, env={}):
|
||||
def __init__(self, env={}, socks5_proxy_url=None):
|
||||
self.env = env
|
||||
self.te = MyTemplateEngine([], env=env)
|
||||
self.hc = HttpClient()
|
||||
self.hc = HttpClient(socks5_proxy_url=socks5_proxy_url)
|
||||
|
||||
async def call(self, url, method='GET', ns={},
|
||||
stream_func=None,
|
||||
@ -234,11 +234,13 @@ class JsonHttpAPI:
|
||||
return ret
|
||||
|
||||
if __name__ == '__main__':
|
||||
async def gbaidu(hc):
|
||||
r = await hc.get('https://www.baidu.com')
|
||||
async def main():
|
||||
hc = HttpClient(socks5_proxy_url='socks5://localhost:1086')
|
||||
r = await hc.request('https://www.baidu.com')
|
||||
print(r)
|
||||
r = await hc.request('https://www.google.com')
|
||||
print(r)
|
||||
await hc.close()
|
||||
loop = asyncio.get_event_loop()
|
||||
hc = HttpClient()
|
||||
loop.run_until_complete(gbaidu(hc))
|
||||
loop.run_until_complete(main())
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user