import asyncio import aiohttp import re from appPublic.log import info, debug, warning, error, exception, critical RESPONSE_BIN = 0 RESPONSE_TEXT = 1 RESPONSE_JSON = 2 RESPONSE_FILE = 3 RESPONSE_STREAM = 4 class HttpError(Exception): def __init__(self, code, msg, *args, **kw): super().__init__(*msg, **kw) self.code = code self.msg = msg def __str__(self): return f"Error Code:{self.code}, {self.msg}" def __expr__(self): return str(self) class HttpClient: def __init__(self,coding='utf-8'): self.coding = coding self.session = None self.cookies = {} async def close(self): if self.session: await self.session.close() self.session = None def url2domain(self,url): parts = url.split('/')[:3] pre = '/'.join(parts) return pre def setCookie(self,url,cookies): name = self.url2domain(url) self.cookies[name] = cookies def getCookies(self,url): name = url2domain(url) return self.cookies.get(name,None) def getsession(self,url): if self.session is None: jar = aiohttp.CookieJar(unsafe=True) self.session = aiohttp.ClientSession(cookie_jar=jar) return self.session async def handleResp(self,url,resp,resp_type, stream_func=None): if resp.cookies is not None: self.setCookie(url,resp.cookies) if resp_type == RESPONSE_BIN: return await resp.read() if resp_type == RESPONSE_JSON: return await resp.json() if resp_type == RESPONSE_TEXT: return await resp.text(self.coding) async for chunk in resp.content.iter_chunked(1024): if stream_func: await stream_func(chunk) def grapCookie(self,url): session = self.getsession(url) domain = self.url2domain(url) filtered = session.cookie_jar.filter_cookies(domain) return filtered async def request(self, url, method, response_type=RESPONSE_TEXT, params=None, data=None, jd=None, stream_func=None, headers=None, **kw): session = self.getsession(url) if params == {}: params = None if data == {}: data = None if jd == {}: jd = None if headers == {}: headers = None resp = await session.request(method, url, params=params, data=data, json=jd, headers=headers, **kw) if resp.status==200: return await self.handleResp(url, resp, response_type, stream_func=stream_func) msg = f'http error({resp.status}, {url=},{params=}, {data=}, {jd=}, {headers=}, {kw=})' exception(msg) raise HttpError(resp.status, msg) async def get(self,url,**kw): return self.request(url, 'GET', **kw) async def post(self,url, **kw): return self.request(url, 'POST', **kw) session = self.getsession(url) if __name__ == '__main__': async def gbaidu(hc): r = await hc.get('https://www.baidu.com') print(r) await hc.close() loop = asyncio.get_event_loop() hc = HttpClient() loop.run_until_complete(gbaidu(hc))