ahserver_g/ahserver/processorResource.py

434 lines
12 KiB
Python
Executable File

import os
import re
import codecs
import aiofiles
from traceback import print_exc
# from showcallstack import showcallstack
import asyncio
import json
from yarl import URL
from aiohttp import client
from aiohttp_auth import auth
from appPublic.http_client import Http_Client
from functools import partial
from aiohttp_auth import auth
from aiohttp.web_urldispatcher import StaticResource, PathLike
from aiohttp.web_urldispatcher import Optional, _ExpectHandler
from aiohttp.web_urldispatcher import Path
from aiohttp.web_response import Response, StreamResponse
from aiohttp.web_exceptions import (
HTTPException,
HTTPExpectationFailed,
HTTPForbidden,
HTTPMethodNotAllowed,
HTTPNotFound,
HTTPFound,
)
from aiohttp.web_fileresponse import FileResponse
from aiohttp.web_request import Request
from aiohttp.web_response import Response, StreamResponse
from aiohttp.web_routedef import AbstractRouteDef
from appPublic.jsonConfig import getConfig
from appPublic.dictObject import DictObject
from appPublic.i18n import getI18N
from appPublic.dictObject import DictObject, multiDict2Dict
from appPublic.timecost import TimeCost
from appPublic.timeUtils import timestampstr
from appPublic.app_logger import AppLogger
from .baseProcessor import getProcessor, BricksUIProcessor, TemplateProcessor
from .baseProcessor import PythonScriptProcessor, MarkdownProcessor
from .xlsxdsProcessor import XLSXDataSourceProcessor
from .llmProcessor import LlmProcessor
from .websocketProcessor import WebsocketProcessor, XtermProcessor
from .sqldsProcessor import SQLDataSourceProcessor
from .functionProcessor import FunctionProcessor
from .proxyProcessor import ProxyProcessor
from .serverenv import ServerEnv
from .url2file import Url2File
from .filestorage import FileStorage, file_realpath
from .restful import DBCrud
from .dbadmin import DBAdmin
from .filedownload import file_download, path_decode
from .utils import unicode_escape
from .filetest import current_fileno
def getHeaderLang(request):
al = request.headers.get('Accept-Language')
if al is None:
return 'en'
return al.split(',')[0]
def i18nDICT(request):
c = getConfig()
i18n = getI18N()
lang = getHeaderLang(request)
l = c.langMapping.get(lang,lang)
return json.dumps(i18n.getLangDict(l)).encode(c.website.coding)
class ProcessorResource(AppLogger, StaticResource,Url2File):
def __init__(self, prefix: str, directory: PathLike,
*, name: Optional[str]=None,
expect_handler: Optional[_ExpectHandler]=None,
chunk_size: int=256 * 1024,
show_index: bool=False, follow_symlinks: bool=False,
append_version: bool=False,
indexes:list=[],
processors:dict={}) -> None:
AppLogger.__init__(self)
StaticResource.__init__(self,prefix, directory,
name=name,
expect_handler=expect_handler,
chunk_size=chunk_size,
show_index=show_index,
follow_symlinks=follow_symlinks,
append_version=append_version)
Url2File.__init__(self,directory,prefix,indexes,inherit=True)
gr = self._routes.get('GET')
self._routes.update({'POST':gr})
self._routes.update({'PUT':gr})
self._routes.update({'OPTIONS':gr})
self._routes.update({'DELETE':gr})
self._routes.update({'TRACE':gr})
self.y_processors = processors
self.y_prefix = prefix
self.y_directory = directory
self.y_indexes = indexes
self.y_env = DictObject()
def setProcessors(self, processors):
self.y_processors = processors
def setIndexes(self, indexes):
self.y_indexes = indexes
def abspath(self, request, path:str):
url = self.entireUrl(request, path)
path = self.url2path(url)
fname = self.url2file(path)
return fname
async def getPostData(self,request: Request) -> DictObject:
qd = {}
if request.query:
qd = multiDict2Dict(request.query)
reader = None
try:
reader = await request.multipart()
except:
# print('reader is None')
pass
if reader is None:
pd = await request.post()
pd = multiDict2Dict(pd)
if pd == {}:
if request.can_read_body:
x = await request.read()
try:
pd = json.loads(x)
except:
# print('body is not a json')
pass
qd.update(pd)
return DictObject(**qd)
ns = qd
while 1:
try:
field = await reader.next()
if not field:
break
value = ''
if hasattr(field,'filename') and field.filename is not None:
saver = FileStorage()
value = await saver.save(field.filename,field.read_chunk)
else:
value = await field.read(decode=True)
value = value.decode('utf-8')
ov = ns.get(field.name)
if ov:
if type(ov) == type([]):
ov.append(value)
else:
ov = [ov,value]
else:
ov = value
ns.update({field.name:ov})
# print(f'getPostData():{ns=}')
except Exception as e:
print(e)
print_exc()
print('-----------except out ------------')
break;
return DictObject(ns)
def parse_request(self, request):
"""
get real schema, host, port, prepath
and save it to self._{attr}
"""
self._scheme = request.scheme
self._scheme = request.headers.get('X-Forwarded-Scheme',request.scheme)
k = request.host.split(':')
host = k[0]
port = 80
if len(k) == 2:
port = int(k[1])
elif self._scheme.lower() == 'https':
port = 443
self._host = request.headers.get('X-Forwarded-Host', host)
self._port = request.headers.get('X-Forwarded-Port', port)
self._prepath = request.headers.get('X-Forwarded-Prepath', '')
if self._prepath != '':
self._prepath = '/' + self._prepath
self._preurl = f'{self._scheme}://{self._host}:{self._port}{self._prepath}'
# print(f'{request.path=}, {self._preurl=}')
async def _handle(self,request:Request) -> StreamResponse:
clientkeys = {
"iPhone":"iphone",
"iPad":"ipad",
"Android":"androidpad",
"Windows Phone":"winphone",
"Windows NT[.]*Win64; x64":"pc",
}
def i18nDICT():
c = getConfig()
g = ServerEnv()
if not g.get('myi18n',False):
g.myi18n = getI18N()
lang = getHeaderLang(request)
l = c.langMapping.get(lang,lang)
return json.dumps(g.myi18n.getLangDict(l))
def getClientType(request):
agent = request.headers.get('user-agent')
if type(agent)!=type('') and type(agent)!=type(b''):
return 'pc'
for k in clientkeys.keys():
m = re.findall(k,agent)
if len(m)>0:
return clientkeys[k]
return 'pc'
def serveri18n(s):
lang = getHeaderLang(request)
c = getConfig()
g = ServerEnv()
if not g.get('myi18n',False):
g.myi18n = getI18N()
l = c.langMapping.get(lang,lang)
return g.myi18n(s,l)
async def getArgs() -> DictObject:
if request.method == 'POST':
return await self.getPostData(request)
ns = multiDict2Dict(request.query)
return DictObject(**ns)
async def redirect(url):
url = self.entireUrl(request, url)
raise HTTPFound(url)
async def remember_user(userid):
await auth.remember(request, userid)
async def remember_ticket(ticket):
await auth.remember_ticket(request, ticket)
async def get_ticket():
return await auth.get_ticket(request)
async def forget_user():
await auth.forget(request)
async def get_user():
return await auth.get_auth(request)
self.parse_request(request)
self.y_env.i18n = serveri18n
self.y_env.file_realpath = file_realpath
self.y_env.redirect = redirect
self.y_env.info = self.info
self.y_env.error = self.error
self.y_env.debug = self.debug
self.y_env.remember_user = remember_user
self.y_env.forget_user = forget_user
self.y_env.get_user = get_user
self.y_env.i18nDict = i18nDICT
self.y_env.terminalType = getClientType(request)
self.y_env.entire_url = partial(self.entireUrl,request)
self.y_env.websocket_url = partial(self.websocketUrl,request)
self.y_env.abspath = self.abspath
self.y_env.request2ns = getArgs
self.y_env.aiohttp_client = client
self.y_env.resource = self
self.y_env.gethost = partial(self.gethost, request)
self.y_env.path_call = partial(self.path_call,request)
self.user = await auth.get_auth(request)
self.y_env.user = self.user
self.request_filename = self.url2file(str(request.path))
request['request_filename'] = self.request_filename
path = request.path
config = getConfig()
request['port'] = config.website.port
if config.website.dbadm and path.startswith(config.website.dbadm):
pp = path.split('/')[2:]
if len(pp)<3:
self.error('%s:not found' % str(request.url))
raise HTTPNotFound
dbname = pp[0]
tablename = pp[1]
action = pp[2]
adm = DBAdmin(request,dbname,tablename,action)
return await adm.render()
if config.website.dbrest and path.startswith(config.website.dbrest):
pp = path.split('/')[2:]
if len(pp)<2:
self.error('%s:not found' % str(request.url))
raise HTTPNotFound
dbname = pp[0]
tablename = pp[1]
id = None
if len(pp) > 2:
id = pp[2]
crud = DBCrud(request,dbname,tablename,id=id)
return await crud.dispatch()
if config.website.download and path.startswith(config.website.download):
pp = path.split('/')[2:]
if len(pp)<1:
self.error('%s:not found' % str(request.url))
raise HTTPNotFound
dp = '/'.join(pp)
path = path_decode(dp)
return await file_download(request, path)
processor = self.url2processor(request, str(request.url), self.request_filename)
if processor:
ret = await processor.handle(request)
return ret
if self.request_filename and await self.isHtml(self.request_filename):
return await self.html_handle(request, self.request_filename)
if self.request_filename and os.path.isdir(self.request_filename):
config = getConfig()
if not config.website.allowListFolder:
self.error('%s:not found' % str(request.url))
raise HTTPNotFound
# print(f'{self.request_filename=}, {str(request.url)=} handle as a normal file')
return await super()._handle(request)
def gethost(self, request):
host = request.headers.get('X-Forwarded-Host')
if host:
return host
host = request.headers.get('Host')
if host:
return host
return '/'.join(str(request.url).split('/')[:3])
async def html_handle(self,request,filepath):
async with aiofiles.open(filepath,'r', encoding='utf-8') as f:
txt = await f.read()
utxt = txt.encode('utf-8')
headers = {
'Content-Type': 'text/html; utf-8',
'Accept-Ranges': 'bytes',
'Content-Length': str(len(utxt))
}
resp = Response(text=txt,headers=headers)
return resp
async def isHtml(self,fn):
try:
async with aiofiles.open(fn,'r',encoding='utf-8') as f:
b = await f.read()
while b[0] in ['\n',' ','\t']:
b = b[1:]
if b.lower().startswith('<html>'):
return True
if b.lower().startswith('<!doctype html>'):
return True
except Exception as e:
return False
def url2processor(self, request, url, fpath):
config = getConfig()
url1 = url
url = self.entireUrl(request, url)
host = '/'.join(url.split('/')[:3])
path = '/' + '/'.join(url.split('/')[3:])
if config.website.startswiths:
for a in config.website.startswiths:
if path.startswith(a.leading):
processor = FunctionProcessor(path,self,a)
return processor
if fpath is None:
print(f'fpath is None ..., {url=}, {url1=}')
return None
for word, handlername in self.y_processors:
if fpath.endswith(word):
Klass = getProcessor(handlername)
try:
processor = Klass(path,self)
# print(f'{f_cnt1=}, {f_cnt2=}, {f_cnt3=}, {f_cnt4=}, {f_cnt5=}')
return processor
except Exception as e:
print('Exception:',e, 'handlername=', handlername)
return None
return None
def websocketUrl(self, request, url):
url = entireUrl(request, url)
if url.startswith('https'):
return 'wss' + url[5:]
return 'ws' + url[4:]
def entireUrl(self, request, url):
if url.startswith('http://') or \
url.startswith('https://') or \
url.startswith('ws://') or \
url.startswith('wss://'):
return url
if url.startswith('/'):
u = f'{self._preurl}{url}'
# print(f'entireUrl(), {u=}, {url=}, {self._preurl=}')
return u
path = request.path
p = self.relatedurl(path,url)
u = f'{self._preurl}{p}'
# print(f'entireUrl(), {u=}, {url=}, {self._preurl=}')
return u
def url2path(self, url):
if url.startswith(self._preurl):
return url[len(self._preurl):]
return url
async def path_call(self, request, path, params={}):
url = self.entireUrl(request, path)
# print(f'{path=}, after entireUrl(), {url=}')
path = self.url2path(url)
fpath = self.url2file(path)
processor = self.url2processor(request, path, fpath)
# print(f'path_call(), {path=}, {url=}, {fpath=}, {processor=}, {self._prepath}')
new_request = request.clone(rel_url=path)
return await processor.be_call(new_request, params=params)