ahserver_g/ahserver/processorResource.py

434 lines
12 KiB
Python
Raw Permalink Normal View History

2019-07-10 17:34:45 +08:00
import os
import re
2020-12-18 14:25:46 +08:00
import codecs
2023-10-12 15:46:05 +08:00
import aiofiles
2020-12-18 13:33:51 +08:00
from traceback import print_exc
2023-02-22 22:16:41 +08:00
# from showcallstack import showcallstack
2019-07-10 17:34:45 +08:00
2020-12-11 01:48:32 +08:00
import asyncio
2023-08-10 12:02:53 +08:00
import json
2020-12-11 01:48:32 +08:00
from yarl import URL
2023-09-05 10:59:09 +08:00
from aiohttp import client
2023-04-04 10:34:02 +08:00
from aiohttp_auth import auth
2020-12-11 01:48:32 +08:00
from appPublic.http_client import Http_Client
from functools import partial
from aiohttp_auth import auth
2022-06-22 16:57:41 +08:00
from aiohttp.web_urldispatcher import StaticResource, PathLike
2019-07-10 17:34:45 +08:00
from aiohttp.web_urldispatcher import Optional, _ExpectHandler
from aiohttp.web_urldispatcher import Path
from aiohttp.web_response import Response, StreamResponse
from aiohttp.web_exceptions import (
2019-11-28 15:24:09 +08:00
HTTPException,
HTTPExpectationFailed,
HTTPForbidden,
HTTPMethodNotAllowed,
HTTPNotFound,
2023-08-04 14:58:12 +08:00
HTTPFound,
2019-07-10 17:34:45 +08:00
)
from aiohttp.web_fileresponse import FileResponse
from aiohttp.web_request import Request
from aiohttp.web_response import Response, StreamResponse
from aiohttp.web_routedef import AbstractRouteDef
from appPublic.jsonConfig import getConfig
2024-04-08 18:12:15 +08:00
from appPublic.dictObject import DictObject
2022-05-20 20:03:54 +08:00
from appPublic.i18n import getI18N
2019-07-29 10:01:11 +08:00
from appPublic.dictObject import DictObject, multiDict2Dict
2020-11-06 11:15:28 +08:00
from appPublic.timecost import TimeCost
2020-11-06 16:53:38 +08:00
from appPublic.timeUtils import timestampstr
2022-06-23 14:58:03 +08:00
from appPublic.app_logger import AppLogger
2019-07-10 17:34:45 +08:00
2024-01-31 17:08:29 +08:00
from .baseProcessor import getProcessor, BricksUIProcessor, TemplateProcessor
from .baseProcessor import PythonScriptProcessor, MarkdownProcessor
2019-07-10 17:34:45 +08:00
from .xlsxdsProcessor import XLSXDataSourceProcessor
2024-04-08 18:12:15 +08:00
from .llmProcessor import LlmProcessor
2023-09-22 13:53:32 +08:00
from .websocketProcessor import WebsocketProcessor, XtermProcessor
2019-07-10 17:34:45 +08:00
from .sqldsProcessor import SQLDataSourceProcessor
2019-11-28 15:24:09 +08:00
from .functionProcessor import FunctionProcessor
2022-03-22 11:45:21 +08:00
from .proxyProcessor import ProxyProcessor
2019-07-10 17:34:45 +08:00
from .serverenv import ServerEnv
from .url2file import Url2File
2023-11-16 16:20:21 +08:00
from .filestorage import FileStorage, file_realpath
2019-07-29 10:01:11 +08:00
from .restful import DBCrud
2020-04-22 11:37:20 +08:00
from .dbadmin import DBAdmin
2019-08-28 10:15:35 +08:00
from .filedownload import file_download, path_decode
2020-12-18 13:33:51 +08:00
from .utils import unicode_escape
2023-12-18 19:09:47 +08:00
from .filetest import current_fileno
2019-07-10 17:34:45 +08:00
def getHeaderLang(request):
al = request.headers.get('Accept-Language')
if al is None:
return 'en'
return al.split(',')[0]
def i18nDICT(request):
c = getConfig()
i18n = getI18N()
lang = getHeaderLang(request)
l = c.langMapping.get(lang,lang)
return json.dumps(i18n.getLangDict(l)).encode(c.website.coding)
2020-11-09 02:39:35 +08:00
2022-06-23 14:58:03 +08:00
class ProcessorResource(AppLogger, StaticResource,Url2File):
2019-07-10 17:34:45 +08:00
def __init__(self, prefix: str, directory: PathLike,
2019-11-28 15:24:09 +08:00
*, name: Optional[str]=None,
expect_handler: Optional[_ExpectHandler]=None,
chunk_size: int=256 * 1024,
show_index: bool=False, follow_symlinks: bool=False,
2019-12-03 11:30:15 +08:00
append_version: bool=False,
indexes:list=[],
2019-12-05 06:04:38 +08:00
processors:dict={}) -> None:
2022-06-23 14:58:03 +08:00
AppLogger.__init__(self)
2019-12-03 11:30:15 +08:00
StaticResource.__init__(self,prefix, directory,
2019-11-28 15:24:09 +08:00
name=name,
expect_handler=expect_handler,
chunk_size=chunk_size,
show_index=show_index,
2019-07-10 17:34:45 +08:00
follow_symlinks=follow_symlinks,
2019-11-28 15:24:09 +08:00
append_version=append_version)
2019-12-03 11:30:15 +08:00
Url2File.__init__(self,directory,prefix,indexes,inherit=True)
2019-07-10 17:34:45 +08:00
gr = self._routes.get('GET')
self._routes.update({'POST':gr})
2019-07-29 10:01:11 +08:00
self._routes.update({'PUT':gr})
self._routes.update({'OPTIONS':gr})
self._routes.update({'DELETE':gr})
self._routes.update({'TRACE':gr})
2019-12-05 06:04:38 +08:00
self.y_processors = processors
2019-07-10 17:34:45 +08:00
self.y_prefix = prefix
self.y_directory = directory
2019-12-05 06:04:38 +08:00
self.y_indexes = indexes
2019-07-10 17:34:45 +08:00
self.y_env = DictObject()
def setProcessors(self, processors):
self.y_processors = processors
def setIndexes(self, indexes):
self.y_indexes = indexes
2024-02-19 11:18:50 +08:00
2020-12-20 11:43:00 +08:00
def abspath(self, request, path:str):
url = self.entireUrl(request, path)
2024-02-19 11:18:50 +08:00
path = self.url2path(url)
fname = self.url2file(path)
2023-04-07 13:16:54 +08:00
return fname
2019-07-10 17:34:45 +08:00
2024-04-08 18:12:15 +08:00
async def getPostData(self,request: Request) -> DictObject:
2023-08-10 17:46:09 +08:00
qd = {}
if request.query:
qd = multiDict2Dict(request.query)
2023-04-18 10:50:55 +08:00
reader = None
try:
reader = await request.multipart()
except:
2023-08-10 17:46:09 +08:00
# print('reader is None')
2023-04-18 10:50:55 +08:00
pass
2019-07-10 17:34:45 +08:00
if reader is None:
2023-08-10 17:46:09 +08:00
pd = await request.post()
pd = multiDict2Dict(pd)
if pd == {}:
if request.can_read_body:
x = await request.read()
try:
pd = json.loads(x)
except:
# print('body is not a json')
pass
qd.update(pd)
2024-04-08 18:12:15 +08:00
return DictObject(**qd)
2023-08-10 17:46:09 +08:00
ns = qd
2019-07-10 17:34:45 +08:00
while 1:
2023-02-22 22:16:41 +08:00
try:
field = await reader.next()
if not field:
break
value = ''
if hasattr(field,'filename') and field.filename is not None:
saver = FileStorage()
value = await saver.save(field.filename,field.read_chunk)
else:
value = await field.read(decode=True)
value = value.decode('utf-8')
ov = ns.get(field.name)
if ov:
if type(ov) == type([]):
ov.append(value)
else:
ov = [ov,value]
2019-07-10 17:34:45 +08:00
else:
2023-02-22 22:16:41 +08:00
ov = value
ns.update({field.name:ov})
# print(f'getPostData():{ns=}')
except Exception as e:
print(e)
print_exc()
print('-----------except out ------------')
break;
2024-04-08 18:12:15 +08:00
return DictObject(ns)
2019-07-10 17:34:45 +08:00
2024-02-18 18:20:52 +08:00
def parse_request(self, request):
"""
get real schema, host, port, prepath
and save it to self._{attr}
"""
self._scheme = request.scheme
self._scheme = request.headers.get('X-Forwarded-Scheme',request.scheme)
k = request.host.split(':')
host = k[0]
port = 80
if len(k) == 2:
port = int(k[1])
elif self._scheme.lower() == 'https':
port = 443
self._host = request.headers.get('X-Forwarded-Host', host)
self._port = request.headers.get('X-Forwarded-Port', port)
self._prepath = request.headers.get('X-Forwarded-Prepath', '')
2024-02-19 11:40:55 +08:00
if self._prepath != '':
self._prepath = '/' + self._prepath
2024-02-19 11:18:50 +08:00
self._preurl = f'{self._scheme}://{self._host}:{self._port}{self._prepath}'
2024-02-19 12:52:50 +08:00
# print(f'{request.path=}, {self._preurl=}')
2024-02-19 11:18:50 +08:00
2024-02-18 18:20:52 +08:00
2019-07-10 17:34:45 +08:00
async def _handle(self,request:Request) -> StreamResponse:
clientkeys = {
"iPhone":"iphone",
"iPad":"ipad",
"Android":"androidpad",
"Windows Phone":"winphone",
"Windows NT[.]*Win64; x64":"pc",
}
def i18nDICT():
c = getConfig()
g = ServerEnv()
if not g.get('myi18n',False):
g.myi18n = getI18N()
lang = getHeaderLang(request)
l = c.langMapping.get(lang,lang)
return json.dumps(g.myi18n.getLangDict(l))
def getClientType(request):
agent = request.headers.get('user-agent')
if type(agent)!=type('') and type(agent)!=type(b''):
return 'pc'
for k in clientkeys.keys():
m = re.findall(k,agent)
if len(m)>0:
return clientkeys[k]
return 'pc'
def serveri18n(s):
lang = getHeaderLang(request)
c = getConfig()
g = ServerEnv()
if not g.get('myi18n',False):
g.myi18n = getI18N()
l = c.langMapping.get(lang,lang)
return g.myi18n(s,l)
2020-12-11 01:48:32 +08:00
2024-04-08 18:12:15 +08:00
async def getArgs() -> DictObject:
2019-07-10 17:34:45 +08:00
if request.method == 'POST':
return await self.getPostData(request)
ns = multiDict2Dict(request.query)
2024-04-08 18:12:15 +08:00
return DictObject(**ns)
2019-07-10 17:34:45 +08:00
2023-08-04 14:58:12 +08:00
async def redirect(url):
url = self.entireUrl(request, url)
raise HTTPFound(url)
2023-04-04 10:34:02 +08:00
async def remember_user(userid):
await auth.remember(request, userid)
2023-06-12 13:41:33 +08:00
async def remember_ticket(ticket):
await auth.remember_ticket(request, ticket)
async def get_ticket():
return await auth.get_ticket(request)
2023-04-04 10:34:02 +08:00
2023-06-07 13:45:18 +08:00
async def forget_user():
await auth.forget(request)
2023-04-04 10:34:02 +08:00
async def get_user():
return await auth.get_auth(request)
2024-02-18 18:20:52 +08:00
self.parse_request(request)
2019-07-10 17:34:45 +08:00
self.y_env.i18n = serveri18n
2023-11-16 16:20:21 +08:00
self.y_env.file_realpath = file_realpath
2023-08-04 14:58:12 +08:00
self.y_env.redirect = redirect
2023-06-08 11:38:12 +08:00
self.y_env.info = self.info
self.y_env.error = self.error
self.y_env.debug = self.debug
2023-04-04 10:34:02 +08:00
self.y_env.remember_user = remember_user
2023-06-07 13:45:18 +08:00
self.y_env.forget_user = forget_user
2023-04-04 10:34:02 +08:00
self.y_env.get_user = get_user
2019-07-10 17:34:45 +08:00
self.y_env.i18nDict = i18nDICT
self.y_env.terminalType = getClientType(request)
2020-12-11 01:48:32 +08:00
self.y_env.entire_url = partial(self.entireUrl,request)
2023-09-22 13:53:32 +08:00
self.y_env.websocket_url = partial(self.websocketUrl,request)
2019-07-10 17:34:45 +08:00
self.y_env.abspath = self.abspath
self.y_env.request2ns = getArgs
2023-09-05 10:59:09 +08:00
self.y_env.aiohttp_client = client
2019-07-10 17:34:45 +08:00
self.y_env.resource = self
2022-04-12 12:11:12 +08:00
self.y_env.gethost = partial(self.gethost, request)
2020-12-16 10:15:43 +08:00
self.y_env.path_call = partial(self.path_call,request)
2020-12-11 01:48:32 +08:00
self.user = await auth.get_auth(request)
2022-01-03 11:53:39 +08:00
self.y_env.user = self.user
2023-04-07 13:16:54 +08:00
self.request_filename = self.url2file(str(request.path))
2023-07-21 17:25:21 +08:00
request['request_filename'] = self.request_filename
2019-07-10 17:34:45 +08:00
path = request.path
2019-07-29 10:01:11 +08:00
config = getConfig()
2023-07-21 17:25:21 +08:00
request['port'] = config.website.port
2020-04-22 11:37:20 +08:00
if config.website.dbadm and path.startswith(config.website.dbadm):
pp = path.split('/')[2:]
if len(pp)<3:
2022-06-23 14:58:03 +08:00
self.error('%s:not found' % str(request.url))
2020-04-22 11:37:20 +08:00
raise HTTPNotFound
dbname = pp[0]
tablename = pp[1]
action = pp[2]
adm = DBAdmin(request,dbname,tablename,action)
return await adm.render()
2019-07-29 10:01:11 +08:00
if config.website.dbrest and path.startswith(config.website.dbrest):
pp = path.split('/')[2:]
if len(pp)<2:
2022-06-23 14:58:03 +08:00
self.error('%s:not found' % str(request.url))
2019-07-29 10:01:11 +08:00
raise HTTPNotFound
dbname = pp[0]
tablename = pp[1]
id = None
if len(pp) > 2:
id = pp[2]
crud = DBCrud(request,dbname,tablename,id=id)
return await crud.dispatch()
2019-08-28 10:15:35 +08:00
if config.website.download and path.startswith(config.website.download):
pp = path.split('/')[2:]
if len(pp)<1:
2022-06-23 14:58:03 +08:00
self.error('%s:not found' % str(request.url))
2019-08-28 10:15:35 +08:00
raise HTTPNotFound
dp = '/'.join(pp)
path = path_decode(dp)
2019-08-28 11:30:31 +08:00
return await file_download(request, path)
2019-11-28 15:24:09 +08:00
2021-05-06 10:09:01 +08:00
processor = self.url2processor(request, str(request.url), self.request_filename)
2020-12-11 01:48:32 +08:00
if processor:
2023-12-18 19:09:47 +08:00
ret = await processor.handle(request)
return ret
2020-12-18 13:27:53 +08:00
2023-10-12 15:46:05 +08:00
if self.request_filename and await self.isHtml(self.request_filename):
2021-05-06 10:09:01 +08:00
return await self.html_handle(request, self.request_filename)
2020-12-18 13:27:53 +08:00
2021-05-06 10:09:01 +08:00
if self.request_filename and os.path.isdir(self.request_filename):
2020-04-05 09:47:13 +08:00
config = getConfig()
if not config.website.allowListFolder:
2022-06-23 14:58:03 +08:00
self.error('%s:not found' % str(request.url))
2020-04-22 11:37:20 +08:00
raise HTTPNotFound
2024-02-18 18:23:40 +08:00
# print(f'{self.request_filename=}, {str(request.url)=} handle as a normal file')
2019-07-10 17:34:45 +08:00
return await super()._handle(request)
2020-12-18 13:27:53 +08:00
2022-04-12 12:11:12 +08:00
def gethost(self, request):
host = request.headers.get('X-Forwarded-Host')
if host:
return host
host = request.headers.get('Host')
if host:
return host
return '/'.join(str(request.url).split('/')[:3])
2020-12-18 14:25:46 +08:00
async def html_handle(self,request,filepath):
2023-10-12 15:46:05 +08:00
async with aiofiles.open(filepath,'r', encoding='utf-8') as f:
txt = await f.read()
2023-10-17 16:07:34 +08:00
utxt = txt.encode('utf-8')
2020-12-18 13:27:53 +08:00
headers = {
'Content-Type': 'text/html; utf-8',
'Accept-Ranges': 'bytes',
2023-10-17 16:07:34 +08:00
'Content-Length': str(len(utxt))
2020-12-18 13:27:53 +08:00
}
2021-06-24 13:56:22 +08:00
resp = Response(text=txt,headers=headers)
2020-12-18 13:27:53 +08:00
return resp
2023-10-12 15:46:05 +08:00
async def isHtml(self,fn):
2020-12-18 13:27:53 +08:00
try:
2023-10-12 15:46:05 +08:00
async with aiofiles.open(fn,'r',encoding='utf-8') as f:
b = await f.read()
2020-12-24 18:55:17 +08:00
while b[0] in ['\n',' ','\t']:
b = b[1:]
if b.lower().startswith('<html>'):
2020-12-18 13:27:53 +08:00
return True
2020-12-24 18:55:17 +08:00
if b.lower().startswith('<!doctype html>'):
2020-12-18 13:27:53 +08:00
return True
2020-12-18 13:33:51 +08:00
except Exception as e:
2020-12-18 13:27:53 +08:00
return False
2019-07-10 17:34:45 +08:00
2021-05-06 10:09:01 +08:00
def url2processor(self, request, url, fpath):
2020-12-16 10:15:43 +08:00
config = getConfig()
2024-02-18 17:34:58 +08:00
url1 = url
2020-12-11 01:48:32 +08:00
url = self.entireUrl(request, url)
2023-07-04 12:02:13 +08:00
host = '/'.join(url.split('/')[:3])
path = '/' + '/'.join(url.split('/')[3:])
2020-12-16 10:15:43 +08:00
if config.website.startswiths:
for a in config.website.startswiths:
if path.startswith(a.leading):
2020-12-20 11:43:00 +08:00
processor = FunctionProcessor(path,self,a)
2020-12-16 10:15:43 +08:00
return processor
2023-05-09 14:43:34 +08:00
if fpath is None:
2024-02-18 17:34:58 +08:00
print(f'fpath is None ..., {url=}, {url1=}')
2023-05-09 14:43:34 +08:00
return None
2020-12-11 01:48:32 +08:00
for word, handlername in self.y_processors:
2021-05-06 10:09:01 +08:00
if fpath.endswith(word):
2020-12-11 01:48:32 +08:00
Klass = getProcessor(handlername)
2023-09-22 13:53:32 +08:00
try:
processor = Klass(path,self)
2023-12-18 22:15:24 +08:00
# print(f'{f_cnt1=}, {f_cnt2=}, {f_cnt3=}, {f_cnt4=}, {f_cnt5=}')
2023-09-22 13:53:32 +08:00
return processor
except Exception as e:
print('Exception:',e, 'handlername=', handlername)
return None
2020-12-11 01:48:32 +08:00
return None
2023-09-22 13:53:32 +08:00
def websocketUrl(self, request, url):
url = entireUrl(request, url)
if url.startswith('https'):
return 'wss' + url[5:]
return 'ws' + url[4:]
2020-12-11 01:48:32 +08:00
def entireUrl(self, request, url):
2023-09-22 13:53:32 +08:00
if url.startswith('http://') or \
url.startswith('https://') or \
url.startswith('ws://') or \
url.startswith('wss://'):
2020-12-11 01:48:32 +08:00
return url
if url.startswith('/'):
2024-02-19 12:06:22 +08:00
u = f'{self._preurl}{url}'
2024-02-19 12:52:50 +08:00
# print(f'entireUrl(), {u=}, {url=}, {self._preurl=}')
2024-02-19 12:06:22 +08:00
return u
2020-12-11 01:48:32 +08:00
path = request.path
p = self.relatedurl(path,url)
2024-02-19 12:06:22 +08:00
u = f'{self._preurl}{p}'
2024-02-19 12:52:50 +08:00
# print(f'entireUrl(), {u=}, {url=}, {self._preurl=}')
2024-02-19 12:06:22 +08:00
return u
2020-12-11 01:48:32 +08:00
2024-02-19 11:18:50 +08:00
def url2path(self, url):
2024-02-19 12:06:22 +08:00
if url.startswith(self._preurl):
return url[len(self._preurl):]
2024-02-19 11:18:50 +08:00
return url
2022-05-12 10:10:59 +08:00
async def path_call(self, request, path, params={}):
2020-12-20 11:43:00 +08:00
url = self.entireUrl(request, path)
2024-02-19 12:52:50 +08:00
# print(f'{path=}, after entireUrl(), {url=}')
2024-02-19 12:31:06 +08:00
path = self.url2path(url)
2024-02-19 11:18:50 +08:00
fpath = self.url2file(path)
2024-02-19 12:06:22 +08:00
processor = self.url2processor(request, path, fpath)
2024-02-19 12:52:50 +08:00
# print(f'path_call(), {path=}, {url=}, {fpath=}, {processor=}, {self._prepath}')
2024-02-19 12:48:20 +08:00
new_request = request.clone(rel_url=path)
2023-07-04 12:02:13 +08:00
return await processor.be_call(new_request, params=params)
2020-12-16 10:15:43 +08:00