This commit is contained in:
yumoqing 2025-05-28 17:31:38 +08:00
parent c638340cd6
commit 982ea828fd

View File

@ -1,7 +1,10 @@
import os import os
from traceback import format_exc
import asyncio import asyncio
import aiohttp import aiohttp
from aiohttp import FormData from aiohttp import FormData
import ssl
import certifi
import json import json
from appPublic.myTE import MyTemplateEngine from appPublic.myTE import MyTemplateEngine
import re import re
@ -87,19 +90,37 @@ class HttpClient:
self.session = aiohttp.ClientSession(cookie_jar=jar) self.session = aiohttp.ClientSession(cookie_jar=jar)
return self.session return self.session
async def handleResp(self,url,resp,resp_type, stream_func=None): async def response_generator(self, url, resp, resp_type=None, stream=False):
if resp.cookies is not None: if resp.cookies is not None:
self.setCookie(url,resp.cookies) self.setCookie(url,resp.cookies)
if stream:
async for chunk in resp.content.iter_chunked(1024):
yield chunk
else:
if resp_type == RESPONSE_BIN:
yield await resp.read()
if resp_type == RESPONSE_JSON:
yield await resp.json()
if resp_type == RESPONSE_TEXT:
yield await resp.text(self.coding)
async def response_handle(self,url, resp, resp_type=None, stream_func=None):
if resp.cookies is not None:
self.setCookie(url,resp.cookies)
if stream_func:
async for chunk in resp.content.iter_chunked(1024):
if stream_func:
await stream_func(chunk)
return None
if resp_type == RESPONSE_BIN: if resp_type == RESPONSE_BIN:
return await resp.read() return await resp.read()
if resp_type == RESPONSE_JSON: if resp_type == RESPONSE_JSON:
return await resp.json() return await resp.json()
if resp_type == RESPONSE_TEXT: if resp_type == RESPONSE_TEXT:
return await resp.text(self.coding) return await resp.text(self.coding)
async for chunk in resp.content.iter_chunked(1024):
if stream_func:
await stream_func(chunk)
def grapCookie(self,url): def grapCookie(self,url):
session = self.getsession(url) session = self.getsession(url)
@ -108,15 +129,12 @@ class HttpClient:
return filtered return filtered
async def make_request(self, url, method='GET', async def make_request(self, url, method='GET',
response_type=RESPONSE_TEXT,
params=None, params=None,
data=None, data=None,
jd=None, jd=None,
stream_func=None,
headers=None, headers=None,
use_proxy=False use_proxy=False
): ):
connector = None
if use_proxy: if use_proxy:
connector = ProxyConnector.from_url(self.socks5_proxy_url) connector = ProxyConnector.from_url(self.socks5_proxy_url)
reco = aiohttp.ClientSession(connector=connector) reco = aiohttp.ClientSession(connector=connector)
@ -132,18 +150,47 @@ class HttpClient:
if headers == {}: if headers == {}:
headers = None headers = None
ssl_ctx = ssl.create_default_context(cafile=certifi.where())
print(f'{method=},{ url=}, {headers=}, {data=}') return await session.request(method, url,
resp = await session.request(method, url,
params=params, params=params,
data=data, data=data,
json=jd, json=jd,
headers=headers) headers=headers,
if resp.status==200: ssl=ssl_ctx
return await self.handleResp(url, resp, response_type, stream_func=stream_func) )
msg = f'http error({resp.status}, {url=},{params=}, {data=}, {jd=})'
exception(msg) async def get_request_response(self, url, method='GET',
raise HttpError(resp.status, msg) params=None,
data=None,
jd=None,
headers=None,
**kw
):
try:
return await self.make_request(url, method=method,
params=params,
data=data,
jd=jd,
use_proxy=False,
headers=headers
)
except:
e = Exception(f'make_request error')
exception(f'{e=}, {format_exc()}')
if self.socks5_proxy_url is None:
raise e
domain = get_domain(url)
debug(f'{self.socks5_proxy_url=}, {self.blocked_domains=}, {domain=}')
if domain not in self.blocked_domains:
self.blocked_domains.add(domain)
self.save_cache()
return await self.make_request(url, method=method,
params=params,
data=data,
jd=jd,
use_proxy=True,
headers=headers
)
async def request(self, url, method='GET', async def request(self, url, method='GET',
response_type=RESPONSE_TEXT, response_type=RESPONSE_TEXT,
@ -154,51 +201,51 @@ class HttpClient:
headers=None, headers=None,
**kw **kw
): ):
if self.socks5_proxy_url is None: resp = await self.get_request_response(url, method=method,
resp = await self.make_request(url, method=method, params=params,
response_type=response_type, data=data,
params=params, jd=jd,
data=data, headers=headers,
jd=jd, **kw
use_proxy=False, )
stream_func=stream_func, if resp.status==200:
headers=headers return await self.response_handle(url, resp,
) resp_type=response_type,
return resp stream_func=stream_func)
domain = get_domain(url)
if domain not in self.blocked_domains: msg = f'http error({resp.status}, {url=},{params=}, {data=}, {jd=})'
try: exception(msg)
resp = await self.make_request(url, method=method, raise HttpError(resp.status, msg)
response_type=response_type,
params=params, async def __call__(self, url, method='GET',
data=data, response_type=RESPONSE_TEXT,
jd=jd, params=None,
use_proxy=False, data=None,
stream_func=stream_func, jd=None,
headers=headers headers=None,
) stream=False,
return resp use_proxy=False,
except: **kw
if domain not in self.blocked_domains: ):
self.blocked_domains.add(domain) resp = await self.get_request_response(url, method=method,
self.save_cache() params=params,
resp = await self.make_request(url, method=method, data=data,
response_type=response_type, jd=jd,
params=params, headers=headers,
data=data, **kw)
jd=jd, if resp.status==200:
use_proxy=True, async for d in self.response_generator(url, resp, resp_type=response_type, stream=stream):
stream_func=stream_func, yield d
headers=headers return
) msg = f'http error({resp.status}, {url=},{params=}, {data=}, {jd=})'
return resp exception(msg)
raise HttpError(resp.status, msg)
async def get(self,url,**kw): async def get(self,url,**kw):
return self.request(url, 'GET', **kw) return self.request(url, 'GET', **kw)
async def post(self,url, **kw): async def post(self,url, **kw):
return self.request(url, 'POST', **kw) return self.request(url, 'POST', **kw)
session = self.getsession(url)
class JsonHttpAPI: class JsonHttpAPI:
def __init__(self, env={}, socks5_proxy_url=None): def __init__(self, env={}, socks5_proxy_url=None):
@ -206,12 +253,73 @@ class JsonHttpAPI:
self.te = MyTemplateEngine([], env=env) self.te = MyTemplateEngine([], env=env)
self.hc = HttpClient(socks5_proxy_url=socks5_proxy_url) self.hc = HttpClient(socks5_proxy_url=socks5_proxy_url)
async def stream_func(self, chunk):
debug(f'{chunk=}')
d = self.chunk_buffer + chuck
a, b = d.split('\n', 1)
self.chunk_buffer = b
if self.resptmpl:
ns1 = json.loads(a)
a = self.te.renders(self.resptmpl, ns1)
if self.user_stream_func:
jd = json.loads(a)
await self.user_stream_func(jd)
async def __call__(self, url, method='GET', ns={},
stream=False,
headerstmpl=None,
paramstmpl=None,
datatmpl=None,
chunk_leading=None,
chunk_end="[done]",
resptmpl=None):
headers = None
ns1 = self.env.copy()
ns1.update(ns)
if headerstmpl:
headers = json.loads(self.te.renders(headerstmpl, ns1))
info(f'{headers=},{ns=}, {headerstmpl=}')
params = None
if paramstmpl:
params = json.loads(self.te.renders(paramstmpl, ns1))
data = None
if datatmpl:
datadic = json.loads(self.te.renders(datatmpl, ns1))
data = json.dumps(datadic, ensure_ascii=False)
"""
data = FormData()
for k,v in datadic.items():
data.add_field(k, v)
headers['Content-Type'] = 'multipart/form-data'
"""
info(f'{data=},{ns=}, {headers=}')
hc = HttpClient()
async for d in self.hc(url, method=method,
stream=stream,
headers=headers,
params=params,
data=data):
if stream:
d = self.chunk_handle(d, chunk_leading, chunk_end)
if isinstance(d, dict) and resptmpl:
ds = self.te.renders(resptmpl, d)
yield json.loads(ds)
else:
yield d
async def call(self, url, method='GET', ns={}, async def call(self, url, method='GET', ns={},
stream_func=None, stream_func=None,
headerstmpl=None, headerstmpl=None,
paramstmpl=None, paramstmpl=None,
datatmpl=None, datatmpl=None,
chunk_leading=None,
chunk_end="[done]",
resptmpl=None): resptmpl=None):
self.user_stream_func = stream_func
self.chunk_leading = chunk_leading
self.chunk_end = chunk_end
self.chunk_buffer = ''
self.resptmpl =resptmpl
headers = None headers = None
ns1 = self.env.copy() ns1 = self.env.copy()
ns1.update(ns) ns1.update(ns)
@ -253,11 +361,11 @@ class JsonHttpAPI:
if __name__ == '__main__': if __name__ == '__main__':
async def main(): async def main():
hc = HttpClient(socks5_proxy_url='socks5://localhost:1086') hc = HttpClient(socks5_proxy_url='socks5://localhost:1086')
r = await hc.request('https://www.baidu.com') async for d in hc('https://www.baidu.com'):
print(r) print(d)
r = await hc.request('https://www.google.com') r = await hc.request('https://www.google.com')
print(r) print(r)
await hc.close() await hc.close()
loop = asyncio.get_event_loop() loop = asyncio.new_event_loop()
loop.run_until_complete(main()) loop.run_until_complete(main())