This commit is contained in:
yumoqing 2019-07-29 09:59:10 +08:00
parent 385d92abc8
commit 60476e2c1f
6 changed files with 439 additions and 7 deletions

83
appPublic/httpclient.py Normal file
View File

@ -0,0 +1,83 @@
import asyncio
import aiohttp
import re
RESPONSE_BIN = 0
RESPONSE_TEXT = 1
RESPONSE_JSON = 2
RESPONSE_FILE = 3
class HttpClient:
def __init__(self,coding='utf-8'):
self.coding = coding
self.session = None
self.cookies = {}
async def close(self):
await self.session.close()
def url2domain(self,url):
parts = url.split('/')[:3]
pre = '/'.join(parts)
return pre
def setCookie(self,url,cookies):
name = self.url2domain(url)
self.cookies[name] = cookies
def getCookies(self,url):
name = url2domain(url)
return self.cookies.get(name,None)
def getsession(self,url):
if self.session is None:
jar = aiohttp.CookieJar(unsafe=True)
self.session = aiohttp.ClientSession(cookie_jar=jar)
return self.session
async def handleResp(self,url,resp,resp_type):
if resp.cookies is not None:
self.setCookie(url,resp.cookies)
if resp_type == RESPONSE_BIN:
return await resp.read()
if resp_type == RESPONSE_JSON:
return await resp.json()
# default resp_type == RESPONSE_TEXT:
return await resp.text(self.coding)
def grapCookie(self,url):
session = self.getsession(url)
domain = self.url2domain(url)
filtered = session.cookie_jar.filter_cookies(domain)
return filtered
print(f'=====domain={domain},cookies={fltered},type={type(fltered)}===')
async def get(self,url,
params={},headers={},resp_type=RESPONSE_TEXT):
session = self.getsession(url)
resp = await session.get(url,params=params,headers=headers)
if resp.status==200:
return await self.handleResp(url,resp,resp_type)
return None
async def post(self,url,
data=b'',headers={},cookies=None,
resp_type=RESPONSE_TEXT):
session = self.getsession(url)
resp = await session.post(url,data=data,
headers=headers,cookies=cookies)
if resp.status==200:
return await self.handleResp(url,resp,resp_type)
print(f'**ERROR**{url} response code={resp.status}')
return None
if __name__ == '__main__':
async def gbaidu(hc):
r = await hc.get('https://www.baidu.com')
print(r)
await hc.close()
loop = asyncio.get_event_loop()
hc = HttpClient()
loop.run_until_complete(gbaidu(hc))

65
appPublic/myTE.py Normal file
View File

@ -0,0 +1,65 @@
import os
import sys
import appPublic.myjson as json
from jinja2 import Environment,FileSystemLoader
import codecs
from appPublic.argsConvert import ArgsConvert
from appPublic.dictObject import DictObject
def isNone(obj):
return obj is None
class MyTemplateEngine:
def __init__(self,pathList,file_coding='utf-8',out_coding='utf-8'):
self.file_coding = file_coding
self.out_coding = out_coding
loader = FileSystemLoader(pathList, encoding=self.file_coding)
self.env = Environment(loader=loader)
denv={
'json':json,
'hasattr':hasattr,
'int':int,
'float':float,
'str':str,
'type':type,
'isNone':isNone,
'len':len,
'recordFind':recordFind,
'render':self.render,
'renders':self.renders,
'ArgsConvert':ArgsConvert,
'renderJsonFile':self.renderJsonFile,
'ospath':lambda x:os.path.sep.join(x.split(os.altsep)),
'basename':lambda x:os.path.basename(x),
'basenameWithoutExt':lambda x:os.path.splitext(os.path.basename(x))[0],
'extname':lambda x:os.path.splitext(x)[-1],
}
self.env.globals.update(denv)
def setGlobal(self,dic):
self.env.globals.update(dic)
def _render(self,template,data):
self._setEnv()
uRet = template.render(**data)
return uRet.encode(self.out_coding)
def renders(self,tmplstring,data):
def getGlobal():
return data
self.set('global',getGlobal)
template = self.env.from_string(tmplstring)
return self._render(template,data)
def render(self,tmplfile,data):
def getGlobal():
return data
self.set('global',getGlobal)
template = self.env.get_template(tmplfile)
return self._render(template,data)
def renderJsonFile(self,tmplfile,jsonfile):
f = codecs.open(jsonfile,"r",self.file_coding)
data = json.load(f)
f.close()
return self.render(tmplfile,data)

62
appPublic/tworkers.py Normal file
View File

@ -0,0 +1,62 @@
import sys
import threading
from threading import Thread
from queue import Queue, Empty
class Worker(Thread):
def __init__(self, rqueue, timeout=1):
Thread.__init__(self)
self.timeout = timeout
self.setDaemon(False)
self.r_queue = rqueue
self.start()
def run(self):
emptyQueue = False
while True:
try:
callable,args,kw = self.r_queue.get(timeout=self.timeout)
if callable is None:
break
callable(*args,**kw)
except Empty:
time.sleep(1)
def resulthandler(self,rez):
pass
class ThreadWorkers:
def __init__(self,num_workers=20):
self.workQueue = Queue()
self.worker_cnt = num_workers
self.workers = []
self.__createThreadPool(num_workers)
def __createThreadPool(self,num):
for i in range(num):
thread = Worker(self.workQueue)
self.workers.append(thread)
def wait_for_complete(self):
for i in range(self.worker_cnt):
self.add_job(None,None,None)
while len(self.workers):
thread = self.workers.pop()
if thread.isAlive():
thread.join()
def add_job(self,callable,args=[],kw={}):
self.workQueue.put([callable,args,kw])
if __name__ == '__main__':
import requests
def get(url):
x = requests.get(url)
print(x.status_code)
tw = ThreadWorkers()
for i in range(10000):
tw.add_job(get,['http://www.baidu.com'])
tw.wait_for_complete()
print('finished')

42
appPublic/worker.py Normal file
View File

@ -0,0 +1,42 @@
import asyncio
from functools import wraps
def asyncCall(func):
@wraps(func)
def wraped_func(*args,**kw):
task = asyncio.ensure_future(func(*args,**kw))
asyncio.gather(task)
return wraped_func
class Worker:
def __init__(self,max=50):
self.semaphore = asyncio.Semaphore(max)
async def __call__(self,callee,*args,**kw):
async with self.semaphore:
return await callee(*args,**kw)
async def run(self,cmd):
async with self.semaphore:
proc = await asyncio.create_subprocess_shell(cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
stdout, stderr = await proc.comunicate()
return stdout, stderr
if __name__ == '__main__':
async def hello(cnt,greeting):
await asyncio.sleep(1)
print(cnt,greeting)
async def run():
w = Worker()
tasks = [ w(hello,i,'hello world') for i in range(1000) ]
await asyncio.wait(tasks)
loop = asyncio.get_event_loop()
loop.run_until_complete(run())

176
appPublic/zmqapi.py Normal file
View File

@ -0,0 +1,176 @@
import asyncio
from collections.abc import Coroutine
# from asyncio.coroutines import iscoroutine
import zmq
import zmq.asyncio
import json
class Publisher:
def __init__(self,port,coding='utf-8',msgid=1000):
self.port = port
self.socket = None
self.coding = coding
self.msgid = msgid
context = zmq.asyncio.Context()
self.socket = context.socket(zmq.PUB)
self.socket.bind('tcp://*:%d' % self.port)
async def publish(self,msg,msgtype='text',msgid=-1):
print(msg,msgtype)
if msgid == -1:
msgid = self.msgid
if msgtype != 'text':
msg = json.dumps(msg)
msgtype = 'json'
s = '%d %s %s' % (msgid,msgtype,msg)
print(s,msgtype,msgid)
b = s.encode(self.coding)
await self.socket.send(b)
def __del__(self):
self.socket.close()
class Subscriber:
def __init__(self,host,ports,msgid,coding='utf-8'):
self.host = host
self.ports = ports
self.msgid = msgid
self.coding = coding
context = zmq.asyncio.Context()
self.socket = context.socket(zmq.SUB)
f = b'%d' % self.msgid
self.socket.setsockopt(zmq.SUBSCRIBE, f)
for p in self.ports:
self.socket.connect("tcp://%s:%d" % (self.host,p))
def addPort(self,port):
self.socket.connect("tcp://%s:%d" % (self.host,port))
#f = b'%d' % self.msgid
#self.socket.setsockopt(zmq.SUBSCRIBE, f)
async def subscribe(self):
ret = await self.socket.recv()
ret = ret.decode(self.coding)
msgid, msgtype, body = ret.split(' ',2)
print('msgid=',msgid,'msgtype=',msgtype,'body=',body)
if msgtype == 'json':
return json.loads(body)
return body
def __del__(self):
self.socket.close()
class RRServer:
"""
a request / response mode server
"""
def __init__(self,port,handler=None):
self.port = port
self.handler = handler
print(type(self.handler))
async def run(self):
running = True
context = zmq.asyncio.Context()
socket = context.socket(zmq.REP)
socket.bind('tcp://*:%s' % self.port)
while running:
rmsg = await socket.recv()
wmsg = rmsg
if self.handler is not None:
wmsg = self.handler(rmsg)
if isinstance(wmsg,Coroutine):
wmsg = await wmsg
await socket.send(wmsg)
socket.close()
class RRClient:
"""
a request / response mode client
"""
def __init__(self,host,port):
self.host = host
self.port = port
context = zmq.asyncio.Context()
self.socket = context.socket(zmq.REQ)
self.socket.connect('tcp://%s:%d' % (self.host,self.port))
async def request(self,msg):
await self.socket.send(msg)
return await self.socket.recv()
class PPPusher:
"""
pusher of Push / Pull mode
"""
def __init__(self,host,port):
self.host = host
self.port = port
context = zmq.asyncio.Context()
self.socket = context.socket(zmq.PUSH)
self.socket.bind('tcp://%s:%d' % (self.host,self.port))
async def push(self,msg):
await self.socket.send(msg)
class PPPuller:
"""
puller of Push / Pull mode
"""
def __init__(self,host,port,handler=None):
self.host = host
self.port = port
self.handler = handler
async def run(self):
self.running = True
context = zmq.asyncio.Context()
socket = context.socket(zmq.PULL)
socket.bind('tcp://%s:%d' % (self.host,self.port))
while self.running:
msg = await self.socket.recv()
if self.handler is not None:
x = self.handler(msg)
if isinstance(x,Coroutine):
await x
class PairClient:
"""
client of Pair mode
"""
def __init__(self,host,port):
self.host = host
self.port = port
context = zmq.asyncio.Context()
self.socket = context.socket(zmq.PAIR)
self.socket.bind('tcp://%s:%d' % (self.host,self.port))
async def request(self,msg):
await self.socket.send(msg)
return await self.socket.recv()
class PairServer:
"""
server of Pair mode
"""
def __init__(self,port,handler=None):
self.port = port
self.handler = handler
self.running = True
async def run(self):
self.running = True
context = zmq.asyncio.Context()
socket = context.socket(zmq.PAIR)
socket.bind('tcp://*:%d' % self.port)
while self.running:
msg = await socket.recv()
ret = msg
if self.handler is not None:
ret = self.handler()
if isinstance(ret,Coroutine):
ret = await ret
await socket.send(ret)

View File

@ -9,14 +9,18 @@ from setuptools import setup, find_packages
# python setup.py bdist_egg generate a egg file
# Release information about eway
version = "5.0.16"
version = "5.0.18"
name = "appPublic"
description = "appPublic"
author = "yumoqing"
email = "yumoqing@icloud.com"
email = "yumoqing@gmail.com"
packages=find_packages()
package_data = {}
with open("README.md", "r") as fh:
long_description = fh.read()
setup(
name="appPublic",
version=version,
@ -32,12 +36,12 @@ setup(
package_data=package_data,
keywords = [
],
url="https://github.com/yumoqing/appPublic",
long_description=long_description,
long_description_content_type="text/markdown",
classifiers = [
'Development Status :: 3 - Alpha',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Topic :: Software Development :: Libraries :: Python Modules',
'Framework :: utils',
'Programming Language :: Python :: 3',
'License :: OSI Approved :: MIT License',
],
platforms= 'any'
)