ahserver_g/ahserver/processorResource.py

370 lines
10 KiB
Python
Raw Normal View History

2019-07-10 17:34:45 +08:00
import os
import re
2020-12-18 14:25:46 +08:00
import codecs
2020-12-18 13:33:51 +08:00
from traceback import print_exc
2023-02-22 22:16:41 +08:00
# from showcallstack import showcallstack
2019-07-10 17:34:45 +08:00
2020-12-11 01:48:32 +08:00
import asyncio
from yarl import URL
2023-04-04 10:34:02 +08:00
from aiohttp_auth import auth
2020-12-11 01:48:32 +08:00
from appPublic.http_client import Http_Client
from functools import partial
from aiohttp_auth import auth
2022-06-22 16:57:41 +08:00
from aiohttp.web_urldispatcher import StaticResource, PathLike
2019-07-10 17:34:45 +08:00
from aiohttp.web_urldispatcher import Optional, _ExpectHandler
from aiohttp.web_urldispatcher import Path
from aiohttp.web_response import Response, StreamResponse
from aiohttp.web_exceptions import (
2019-11-28 15:24:09 +08:00
HTTPException,
HTTPExpectationFailed,
HTTPForbidden,
HTTPMethodNotAllowed,
HTTPNotFound,
2019-07-10 17:34:45 +08:00
)
from aiohttp.web_fileresponse import FileResponse
from aiohttp.web_request import Request
from aiohttp.web_response import Response, StreamResponse
from aiohttp.web_routedef import AbstractRouteDef
from appPublic.jsonConfig import getConfig
2022-05-20 20:03:54 +08:00
from appPublic.i18n import getI18N
2019-07-29 10:01:11 +08:00
from appPublic.dictObject import DictObject, multiDict2Dict
2020-11-06 11:15:28 +08:00
from appPublic.timecost import TimeCost
2020-11-06 16:53:38 +08:00
from appPublic.timeUtils import timestampstr
2022-06-23 14:58:03 +08:00
from appPublic.app_logger import AppLogger
2019-07-10 17:34:45 +08:00
from .baseProcessor import getProcessor
from .xlsxdsProcessor import XLSXDataSourceProcessor
2023-04-14 04:26:41 +08:00
from .websocketProcessor import WebsocketProcessor
2019-07-10 17:34:45 +08:00
from .sqldsProcessor import SQLDataSourceProcessor
2019-11-28 15:24:09 +08:00
from .functionProcessor import FunctionProcessor
2022-03-22 11:45:21 +08:00
from .proxyProcessor import ProxyProcessor
2019-07-10 17:34:45 +08:00
from .serverenv import ServerEnv
from .url2file import Url2File
from .filestorage import FileStorage
2019-07-29 10:01:11 +08:00
from .restful import DBCrud
2020-04-22 11:37:20 +08:00
from .dbadmin import DBAdmin
2019-08-28 10:15:35 +08:00
from .filedownload import file_download, path_decode
2020-12-18 13:33:51 +08:00
from .utils import unicode_escape
2019-07-10 17:34:45 +08:00
def getHeaderLang(request):
al = request.headers.get('Accept-Language')
if al is None:
return 'en'
return al.split(',')[0]
def i18nDICT(request):
c = getConfig()
i18n = getI18N()
lang = getHeaderLang(request)
l = c.langMapping.get(lang,lang)
return json.dumps(i18n.getLangDict(l)).encode(c.website.coding)
2020-11-09 02:39:35 +08:00
2022-06-23 14:58:03 +08:00
class ProcessorResource(AppLogger, StaticResource,Url2File):
2019-07-10 17:34:45 +08:00
def __init__(self, prefix: str, directory: PathLike,
2019-11-28 15:24:09 +08:00
*, name: Optional[str]=None,
expect_handler: Optional[_ExpectHandler]=None,
chunk_size: int=256 * 1024,
show_index: bool=False, follow_symlinks: bool=False,
2019-12-03 11:30:15 +08:00
append_version: bool=False,
indexes:list=[],
2019-12-05 06:04:38 +08:00
processors:dict={}) -> None:
2022-06-23 14:58:03 +08:00
AppLogger.__init__(self)
2019-12-03 11:30:15 +08:00
StaticResource.__init__(self,prefix, directory,
2019-11-28 15:24:09 +08:00
name=name,
expect_handler=expect_handler,
chunk_size=chunk_size,
show_index=show_index,
2019-07-10 17:34:45 +08:00
follow_symlinks=follow_symlinks,
2019-11-28 15:24:09 +08:00
append_version=append_version)
2019-12-03 11:30:15 +08:00
Url2File.__init__(self,directory,prefix,indexes,inherit=True)
2019-07-10 17:34:45 +08:00
gr = self._routes.get('GET')
self._routes.update({'POST':gr})
2019-07-29 10:01:11 +08:00
self._routes.update({'PUT':gr})
self._routes.update({'OPTIONS':gr})
self._routes.update({'DELETE':gr})
self._routes.update({'TRACE':gr})
2019-12-05 06:04:38 +08:00
self.y_processors = processors
2019-07-10 17:34:45 +08:00
self.y_prefix = prefix
self.y_directory = directory
2019-12-05 06:04:38 +08:00
self.y_indexes = indexes
2019-07-10 17:34:45 +08:00
self.y_env = DictObject()
def setProcessors(self, processors):
self.y_processors = processors
def setIndexes(self, indexes):
self.y_indexes = indexes
2020-12-20 11:43:00 +08:00
def abspath(self, request, path:str):
url = self.entireUrl(request, path)
2023-04-07 13:16:54 +08:00
fname = self.url2file(url)
return fname
2019-07-10 17:34:45 +08:00
2019-07-29 10:01:11 +08:00
async def getPostData(self,request: Request) -> dict:
2023-04-18 10:50:55 +08:00
reader = None
try:
reader = await request.multipart()
except:
pass
2019-07-10 17:34:45 +08:00
if reader is None:
md = await request.post()
2023-06-22 20:25:57 +08:00
if md == {}:
if request.query:
return multiDict2Dict(request.query)
2019-07-10 17:34:45 +08:00
ns = multiDict2Dict(md)
2023-06-22 20:25:57 +08:00
print(f'{ns=} reader is None, {request.query=}')
2019-07-10 17:34:45 +08:00
return ns
ns = {}
while 1:
2023-02-22 22:16:41 +08:00
try:
field = await reader.next()
if not field:
break
value = ''
if hasattr(field,'filename') and field.filename is not None:
saver = FileStorage()
value = await saver.save(field.filename,field.read_chunk)
else:
value = await field.read(decode=True)
value = value.decode('utf-8')
ov = ns.get(field.name)
if ov:
if type(ov) == type([]):
ov.append(value)
else:
ov = [ov,value]
2019-07-10 17:34:45 +08:00
else:
2023-02-22 22:16:41 +08:00
ov = value
ns.update({field.name:ov})
# print(f'getPostData():{ns=}')
except Exception as e:
print(e)
print_exc()
print('-----------except out ------------')
break;
# showcallstack()
2023-06-22 20:25:57 +08:00
print(f'getPostData():{ns=}')
2019-07-10 17:34:45 +08:00
return ns
async def _handle(self,request:Request) -> StreamResponse:
2020-11-06 11:15:28 +08:00
name = str(request.url)
2020-11-06 11:27:42 +08:00
t = TimeCost(name)
with t:
2022-06-23 14:58:03 +08:00
try:
x = await self._handle1(request)
2022-06-23 16:43:52 +08:00
except Exception as e:
self.error(f'{name}:error={e}')
2022-06-23 23:31:00 +08:00
print_exc()
2022-06-23 16:43:52 +08:00
x = e
2022-06-23 14:58:03 +08:00
self.info(f'{name}:time cost={t.end_time - t.begin_time}')
2022-06-23 23:21:27 +08:00
if not isinstance(x, StreamResponse):
return HTTPException
2020-11-06 11:15:28 +08:00
return x
async def _handle1(self,request:Request) -> StreamResponse:
2019-07-10 17:34:45 +08:00
clientkeys = {
"iPhone":"iphone",
"iPad":"ipad",
"Android":"androidpad",
"Windows Phone":"winphone",
"Windows NT[.]*Win64; x64":"pc",
}
def i18nDICT():
c = getConfig()
g = ServerEnv()
if not g.get('myi18n',False):
g.myi18n = getI18N()
lang = getHeaderLang(request)
l = c.langMapping.get(lang,lang)
return json.dumps(g.myi18n.getLangDict(l))
def getClientType(request):
agent = request.headers.get('user-agent')
if type(agent)!=type('') and type(agent)!=type(b''):
return 'pc'
for k in clientkeys.keys():
m = re.findall(k,agent)
if len(m)>0:
return clientkeys[k]
return 'pc'
def serveri18n(s):
lang = getHeaderLang(request)
c = getConfig()
g = ServerEnv()
if not g.get('myi18n',False):
g.myi18n = getI18N()
l = c.langMapping.get(lang,lang)
return g.myi18n(s,l)
2020-12-11 01:48:32 +08:00
2019-07-10 17:34:45 +08:00
async def getArgs():
ns = DictObject()
if request.method == 'POST':
return await self.getPostData(request)
ns = multiDict2Dict(request.query)
return ns
2023-04-04 10:34:02 +08:00
async def remember_user(userid):
await auth.remember(request, userid)
2023-06-12 13:41:33 +08:00
async def remember_ticket(ticket):
await auth.remember_ticket(request, ticket)
async def get_ticket():
return await auth.get_ticket(request)
2023-04-04 10:34:02 +08:00
2023-06-07 13:45:18 +08:00
async def forget_user():
await auth.forget(request)
2023-04-04 10:34:02 +08:00
async def get_user():
return await auth.get_auth(request)
2019-07-10 17:34:45 +08:00
self.y_env.i18n = serveri18n
2023-06-08 11:38:12 +08:00
self.y_env.info = self.info
self.y_env.error = self.error
self.y_env.debug = self.debug
2023-04-04 10:34:02 +08:00
self.y_env.remember_user = remember_user
2023-06-07 13:45:18 +08:00
self.y_env.forget_user = forget_user
2023-04-04 10:34:02 +08:00
self.y_env.get_user = get_user
2019-07-10 17:34:45 +08:00
self.y_env.i18nDict = i18nDICT
self.y_env.terminalType = getClientType(request)
2020-12-11 01:48:32 +08:00
self.y_env.entire_url = partial(self.entireUrl,request)
2019-07-10 17:34:45 +08:00
self.y_env.abspath = self.abspath
self.y_env.request2ns = getArgs
self.y_env.resource = self
2022-04-12 12:11:12 +08:00
self.y_env.gethost = partial(self.gethost, request)
2020-12-16 10:15:43 +08:00
self.y_env.path_call = partial(self.path_call,request)
2020-12-11 01:48:32 +08:00
self.user = await auth.get_auth(request)
2022-01-03 11:53:39 +08:00
self.y_env.user = self.user
2023-04-07 13:16:54 +08:00
self.request_filename = self.url2file(str(request.path))
2019-07-10 17:34:45 +08:00
path = request.path
2019-07-29 10:01:11 +08:00
config = getConfig()
2020-04-22 11:37:20 +08:00
if config.website.dbadm and path.startswith(config.website.dbadm):
pp = path.split('/')[2:]
if len(pp)<3:
2022-06-23 14:58:03 +08:00
self.error('%s:not found' % str(request.url))
2020-04-22 11:37:20 +08:00
raise HTTPNotFound
dbname = pp[0]
tablename = pp[1]
action = pp[2]
adm = DBAdmin(request,dbname,tablename,action)
return await adm.render()
2019-07-29 10:01:11 +08:00
if config.website.dbrest and path.startswith(config.website.dbrest):
pp = path.split('/')[2:]
if len(pp)<2:
2022-06-23 14:58:03 +08:00
self.error('%s:not found' % str(request.url))
2019-07-29 10:01:11 +08:00
raise HTTPNotFound
dbname = pp[0]
tablename = pp[1]
id = None
if len(pp) > 2:
id = pp[2]
crud = DBCrud(request,dbname,tablename,id=id)
return await crud.dispatch()
2019-08-28 10:15:35 +08:00
if config.website.download and path.startswith(config.website.download):
pp = path.split('/')[2:]
if len(pp)<1:
2022-06-23 14:58:03 +08:00
self.error('%s:not found' % str(request.url))
2019-08-28 10:15:35 +08:00
raise HTTPNotFound
dp = '/'.join(pp)
path = path_decode(dp)
2019-08-28 11:30:31 +08:00
return await file_download(request, path)
2019-11-28 15:24:09 +08:00
2021-05-06 10:09:01 +08:00
# processor = self.url2processor(request, str(request.url))
processor = self.url2processor(request, str(request.url), self.request_filename)
2020-12-11 01:48:32 +08:00
if processor:
return await processor.handle(request)
2020-12-18 13:27:53 +08:00
2021-05-06 10:09:01 +08:00
if self.request_filename and self.isHtml(self.request_filename):
return await self.html_handle(request, self.request_filename)
2020-12-18 13:27:53 +08:00
2021-05-06 10:09:01 +08:00
if self.request_filename and os.path.isdir(self.request_filename):
2020-04-05 09:47:13 +08:00
config = getConfig()
if not config.website.allowListFolder:
2022-06-23 14:58:03 +08:00
self.error('%s:not found' % str(request.url))
2020-04-22 11:37:20 +08:00
raise HTTPNotFound
2019-07-10 17:34:45 +08:00
return await super()._handle(request)
2020-12-18 13:27:53 +08:00
2022-04-12 12:11:12 +08:00
def gethost(self, request):
host = request.headers.get('X-Forwarded-Host')
if host:
return host
host = request.headers.get('Host')
if host:
return host
return '/'.join(str(request.url).split('/')[:3])
2020-12-18 14:25:46 +08:00
async def html_handle(self,request,filepath):
2021-06-24 13:56:22 +08:00
with open(filepath,'rb') as f:
2020-12-18 13:27:53 +08:00
b = f.read()
2021-06-24 13:56:22 +08:00
utxt = b.decode('unicode_escape')
txt = b.decode('utf-8')
2020-12-18 13:27:53 +08:00
headers = {
'Content-Type': 'text/html; utf-8',
'Accept-Ranges': 'bytes',
2021-06-24 13:56:22 +08:00
'Content-Length': str(len(utxt))
2020-12-18 13:27:53 +08:00
}
2021-06-24 13:56:22 +08:00
resp = Response(text=txt,headers=headers)
2020-12-18 13:27:53 +08:00
return resp
def isHtml(self,fn):
try:
with codecs.open(fn,'r','utf-8') as f:
b = f.read()
2020-12-24 18:55:17 +08:00
while b[0] in ['\n',' ','\t']:
b = b[1:]
if b.lower().startswith('<html>'):
2020-12-18 13:27:53 +08:00
return True
2020-12-24 18:55:17 +08:00
if b.lower().startswith('<!doctype html>'):
2020-12-18 13:27:53 +08:00
return True
2020-12-18 13:33:51 +08:00
except Exception as e:
2020-12-18 13:27:53 +08:00
return False
2019-07-10 17:34:45 +08:00
2021-05-06 10:09:01 +08:00
def url2processor(self, request, url, fpath):
2020-12-16 10:15:43 +08:00
config = getConfig()
2020-12-11 01:48:32 +08:00
url = self.entireUrl(request, url)
host = '/'.join(str(request.url).split('/')[:3])
2023-05-09 14:43:34 +08:00
path = request.path
2020-12-20 11:43:00 +08:00
real_path = self.abspath(request, path)
2020-12-16 10:15:43 +08:00
if config.website.startswiths:
for a in config.website.startswiths:
if path.startswith(a.leading):
2020-12-20 11:43:00 +08:00
processor = FunctionProcessor(path,self,a)
2020-12-16 10:15:43 +08:00
return processor
2023-05-09 14:43:34 +08:00
if real_path is None:
return None
if fpath is None:
return None
2020-12-11 01:48:32 +08:00
for word, handlername in self.y_processors:
2021-05-06 10:09:01 +08:00
if fpath.endswith(word):
2020-12-11 01:48:32 +08:00
Klass = getProcessor(handlername)
2020-12-20 11:43:00 +08:00
processor = Klass(path,self)
2020-12-11 01:48:32 +08:00
return processor
return None
def entireUrl(self, request, url):
if url.startswith('http://') or url.startswith('https://'):
return url
2022-04-12 12:11:12 +08:00
h = self.gethost(request)
2020-12-11 01:48:32 +08:00
if url.startswith('/'):
2022-04-12 12:11:12 +08:00
return '%s://%s%s' % (request.scheme, h, url)
2020-12-11 01:48:32 +08:00
path = request.path
2021-05-06 10:09:01 +08:00
if self.request_filename and os.path.isdir(self.request_filename):
2020-12-24 18:55:17 +08:00
path = '%s/oops' % path
2020-12-11 01:48:32 +08:00
p = self.relatedurl(path,url)
2022-04-12 12:11:12 +08:00
return '%s://%s%s' % (request.scheme, h, p)
2020-12-11 01:48:32 +08:00
2022-05-12 10:10:59 +08:00
async def path_call(self, request, path, params={}):
2020-12-20 11:43:00 +08:00
url = self.entireUrl(request, path)
2021-05-06 10:09:01 +08:00
fpath = self.url2file(url)
processor = self.url2processor(request, url, fpath)
2020-12-20 14:32:08 +08:00
return await processor.path_call(request, params=params)
2020-12-16 10:15:43 +08:00