This commit is contained in:
yumoqing 2025-03-06 01:02:28 +08:00
parent e5d72b67bf
commit ec090a205e

View File

@ -1,3 +1,4 @@
import os
import asyncio
import aiohttp
from aiohttp import FormData
@ -39,7 +40,6 @@ class HttpClient:
self.coding = coding
self.session = None
self.cookies = {}
self.socks5proxy = socks5proxy
self.proxy_connector = None
self.socks5_proxy_url = socks5_proxy_url
self.blocked_domains = set()
@ -107,25 +107,25 @@ class HttpClient:
filtered = session.cookie_jar.filter_cookies(domain)
return filtered
async def make_request(self, url, method,
async def make_request(self, url, method='GET',
response_type=RESPONSE_TEXT,
params=None,
data=None,
jd=None,
stream_func=None,
headers=None,
use_proxy=False,
**kw):
async with aiohttp.ClientSession() as session:
use_proxy=False
):
connector = None
if use_proxy:
connector = ProxyConnector.from_url(self.socks5_proxy_url)
async with aiohttp.ClientSession(connector=connector) as session:
if params == {}:
params = None
if data == {}:
data = None
if jd == {}:
jd = None
proxy = None
if use_proxy and self.socks5_proxy_url:
proxy = self.socks5_proxy_url
if headers == {}:
headers = None
@ -134,15 +134,14 @@ class HttpClient:
params=params,
data=data,
json=jd,
proxy=proxy,
headers=headers, **kw)
headers=headers)
if resp.status==200:
return await self.handleResp(url, resp, response_type, stream_func=stream_func)
msg = f'http error({resp.status}, {url=},{params=}, {data=}, {jd=}, {headers=}, {kw=})'
exception(msg)
raise HttpError(resp.status, msg)
async def request(self, url, method,
async def request(self, url, method='GET',
response_type=RESPONSE_TEXT,
params=None,
data=None,
@ -152,41 +151,42 @@ class HttpClient:
**kw
):
if self.socks5_proxy_url is None:
resp = await self.make_request(url, method,
resp = await self.make_request(url, method=method,
response_type=response_type,
params=params,
data=data,
jd=jd,
proxy=False,
use_proxy=False,
stream_func=stream_func,
headers=headers,
**kw)
headers=headers
)
return resp
domain = get_domain(url)
if domain not in self.blocked_domains:
try:
resp = await self.make_request(url, method,
resp = await self.make_request(url, method=method,
response_type=response_type,
params=params,
data=data,
jd=jd,
proxy=False,
use_proxy=False,
stream_func=stream_func,
headers=headers,
**kw)
headers=headers
)
return resp
except aiohttp.ClientError:
except:
if domain not in self.blocked_domains:
self.blocked_domains.add(domain)
self.save_cache()
resp = await self.make_request(url, method,
resp = await self.make_request(url, method=method,
response_type=response_type,
params=params,
data=data,
jd=jd,
proxy=True,
use_proxy=True,
stream_func=stream_func,
headers=headers,
**kw)
headers=headers
)
return resp
async def get(self,url,**kw):
@ -197,10 +197,10 @@ class HttpClient:
session = self.getsession(url)
class JsonHttpAPI:
def __init__(self, env={}):
def __init__(self, env={}, socks5_proxy_url=None):
self.env = env
self.te = MyTemplateEngine([], env=env)
self.hc = HttpClient()
self.hc = HttpClient(socks5_proxy_url=socks5_proxy_url)
async def call(self, url, method='GET', ns={},
stream_func=None,
@ -234,11 +234,13 @@ class JsonHttpAPI:
return ret
if __name__ == '__main__':
async def gbaidu(hc):
r = await hc.get('https://www.baidu.com')
async def main():
hc = HttpClient(socks5_proxy_url='socks5://localhost:1086')
r = await hc.request('https://www.baidu.com')
print(r)
r = await hc.request('https://www.google.com')
print(r)
await hc.close()
loop = asyncio.get_event_loop()
hc = HttpClient()
loop.run_until_complete(gbaidu(hc))
loop.run_until_complete(main())