import os import re import codecs import aiofiles from traceback import print_exc # from showcallstack import showcallstack import asyncio import json from yarl import URL from aiohttp import client from aiohttp_auth import auth from appPublic.http_client import Http_Client from functools import partial from aiohttp_auth import auth from aiohttp.web_urldispatcher import StaticResource, PathLike from aiohttp.web_urldispatcher import Optional, _ExpectHandler from aiohttp.web_urldispatcher import Path from aiohttp.web_response import Response, StreamResponse from aiohttp.web_exceptions import ( HTTPException, HTTPExpectationFailed, HTTPForbidden, HTTPMethodNotAllowed, HTTPNotFound, HTTPFound, ) from aiohttp.web_fileresponse import FileResponse from aiohttp.web_request import Request from aiohttp.web_response import Response, StreamResponse from aiohttp.web_routedef import AbstractRouteDef from appPublic.jsonConfig import getConfig from appPublic.dictObject import DictObject from appPublic.i18n import getI18N from appPublic.dictObject import DictObject, multiDict2Dict from appPublic.timecost import TimeCost from appPublic.timeUtils import timestampstr from appPublic.app_logger import AppLogger from .baseProcessor import getProcessor, BricksUIProcessor, TemplateProcessor from .baseProcessor import PythonScriptProcessor, MarkdownProcessor from .xlsxdsProcessor import XLSXDataSourceProcessor from .llmProcessor import LlmProcessor from .websocketProcessor import WebsocketProcessor, XtermProcessor from .sqldsProcessor import SQLDataSourceProcessor from .functionProcessor import FunctionProcessor from .proxyProcessor import ProxyProcessor from .serverenv import ServerEnv from .url2file import Url2File from .filestorage import FileStorage, file_realpath from .restful import DBCrud from .dbadmin import DBAdmin from .filedownload import file_download, path_decode from .utils import unicode_escape from .filetest import current_fileno def getHeaderLang(request): al = request.headers.get('Accept-Language') if al is None: return 'en' return al.split(',')[0] def i18nDICT(request): c = getConfig() i18n = getI18N() lang = getHeaderLang(request) l = c.langMapping.get(lang,lang) return json.dumps(i18n.getLangDict(l)).encode(c.website.coding) class ProcessorResource(AppLogger, StaticResource,Url2File): def __init__(self, prefix: str, directory: PathLike, *, name: Optional[str]=None, expect_handler: Optional[_ExpectHandler]=None, chunk_size: int=256 * 1024, show_index: bool=False, follow_symlinks: bool=False, append_version: bool=False, indexes:list=[], processors:dict={}) -> None: AppLogger.__init__(self) StaticResource.__init__(self,prefix, directory, name=name, expect_handler=expect_handler, chunk_size=chunk_size, show_index=show_index, follow_symlinks=follow_symlinks, append_version=append_version) Url2File.__init__(self,directory,prefix,indexes,inherit=True) gr = self._routes.get('GET') self._routes.update({'POST':gr}) self._routes.update({'PUT':gr}) self._routes.update({'OPTIONS':gr}) self._routes.update({'DELETE':gr}) self._routes.update({'TRACE':gr}) self.y_processors = processors self.y_prefix = prefix self.y_directory = directory self.y_indexes = indexes self.y_env = DictObject() def setProcessors(self, processors): self.y_processors = processors def setIndexes(self, indexes): self.y_indexes = indexes def abspath(self, request, path:str): url = self.entireUrl(request, path) path = self.url2path(url) fname = self.url2file(path) return fname async def getPostData(self,request: Request) -> DictObject: qd = {} if request.query: qd = multiDict2Dict(request.query) reader = None try: reader = await request.multipart() except: # print('reader is None') pass if reader is None: pd = await request.post() pd = multiDict2Dict(pd) if pd == {}: if request.can_read_body: x = await request.read() try: pd = json.loads(x) except: # print('body is not a json') pass qd.update(pd) return DictObject(**qd) ns = qd while 1: try: field = await reader.next() if not field: break value = '' if hasattr(field,'filename') and field.filename is not None: saver = FileStorage() value = await saver.save(field.filename,field.read_chunk) else: value = await field.read(decode=True) value = value.decode('utf-8') ov = ns.get(field.name) if ov: if type(ov) == type([]): ov.append(value) else: ov = [ov,value] else: ov = value ns.update({field.name:ov}) # print(f'getPostData():{ns=}') except Exception as e: print(e) print_exc() print('-----------except out ------------') break; return DictObject(ns) def parse_request(self, request): """ get real schema, host, port, prepath and save it to self._{attr} """ self._scheme = request.scheme self._scheme = request.headers.get('X-Forwarded-Scheme',request.scheme) k = request.host.split(':') host = k[0] port = 80 if len(k) == 2: port = int(k[1]) elif self._scheme.lower() == 'https': port = 443 self._host = request.headers.get('X-Forwarded-Host', host) self._port = request.headers.get('X-Forwarded-Port', port) self._prepath = request.headers.get('X-Forwarded-Prepath', '') if self._prepath != '': self._prepath = '/' + self._prepath self._preurl = f'{self._scheme}://{self._host}:{self._port}{self._prepath}' # print(f'{request.path=}, {self._preurl=}') async def _handle(self,request:Request) -> StreamResponse: clientkeys = { "iPhone":"iphone", "iPad":"ipad", "Android":"androidpad", "Windows Phone":"winphone", "Windows NT[.]*Win64; x64":"pc", } def i18nDICT(): c = getConfig() g = ServerEnv() if not g.get('myi18n',False): g.myi18n = getI18N() lang = getHeaderLang(request) l = c.langMapping.get(lang,lang) return json.dumps(g.myi18n.getLangDict(l)) def getClientType(request): agent = request.headers.get('user-agent') if type(agent)!=type('') and type(agent)!=type(b''): return 'pc' for k in clientkeys.keys(): m = re.findall(k,agent) if len(m)>0: return clientkeys[k] return 'pc' def serveri18n(s): lang = getHeaderLang(request) c = getConfig() g = ServerEnv() if not g.get('myi18n',False): g.myi18n = getI18N() l = c.langMapping.get(lang,lang) return g.myi18n(s,l) async def getArgs() -> DictObject: if request.method == 'POST': return await self.getPostData(request) ns = multiDict2Dict(request.query) return DictObject(**ns) async def redirect(url): url = self.entireUrl(request, url) raise HTTPFound(url) async def remember_user(userid): await auth.remember(request, userid) async def remember_ticket(ticket): await auth.remember_ticket(request, ticket) async def get_ticket(): return await auth.get_ticket(request) async def forget_user(): await auth.forget(request) async def get_user(): return await auth.get_auth(request) self.parse_request(request) self.y_env.i18n = serveri18n self.y_env.file_realpath = file_realpath self.y_env.redirect = redirect self.y_env.info = self.info self.y_env.error = self.error self.y_env.debug = self.debug self.y_env.remember_user = remember_user self.y_env.forget_user = forget_user self.y_env.get_user = get_user self.y_env.i18nDict = i18nDICT self.y_env.terminalType = getClientType(request) self.y_env.entire_url = partial(self.entireUrl,request) self.y_env.websocket_url = partial(self.websocketUrl,request) self.y_env.abspath = self.abspath self.y_env.request2ns = getArgs self.y_env.aiohttp_client = client self.y_env.resource = self self.y_env.gethost = partial(self.gethost, request) self.y_env.path_call = partial(self.path_call,request) self.user = await auth.get_auth(request) self.y_env.user = self.user self.request_filename = self.url2file(str(request.path)) request['request_filename'] = self.request_filename path = request.path config = getConfig() request['port'] = config.website.port if config.website.dbadm and path.startswith(config.website.dbadm): pp = path.split('/')[2:] if len(pp)<3: self.error('%s:not found' % str(request.url)) raise HTTPNotFound dbname = pp[0] tablename = pp[1] action = pp[2] adm = DBAdmin(request,dbname,tablename,action) return await adm.render() if config.website.dbrest and path.startswith(config.website.dbrest): pp = path.split('/')[2:] if len(pp)<2: self.error('%s:not found' % str(request.url)) raise HTTPNotFound dbname = pp[0] tablename = pp[1] id = None if len(pp) > 2: id = pp[2] crud = DBCrud(request,dbname,tablename,id=id) return await crud.dispatch() if config.website.download and path.startswith(config.website.download): pp = path.split('/')[2:] if len(pp)<1: self.error('%s:not found' % str(request.url)) raise HTTPNotFound dp = '/'.join(pp) path = path_decode(dp) return await file_download(request, path) processor = self.url2processor(request, str(request.url), self.request_filename) if processor: ret = await processor.handle(request) return ret if self.request_filename and await self.isHtml(self.request_filename): return await self.html_handle(request, self.request_filename) if self.request_filename and os.path.isdir(self.request_filename): config = getConfig() if not config.website.allowListFolder: self.error('%s:not found' % str(request.url)) raise HTTPNotFound # print(f'{self.request_filename=}, {str(request.url)=} handle as a normal file') return await super()._handle(request) def gethost(self, request): host = request.headers.get('X-Forwarded-Host') if host: return host host = request.headers.get('Host') if host: return host return '/'.join(str(request.url).split('/')[:3]) async def html_handle(self,request,filepath): async with aiofiles.open(filepath,'r', encoding='utf-8') as f: txt = await f.read() utxt = txt.encode('utf-8') headers = { 'Content-Type': 'text/html; utf-8', 'Accept-Ranges': 'bytes', 'Content-Length': str(len(utxt)) } resp = Response(text=txt,headers=headers) return resp async def isHtml(self,fn): try: async with aiofiles.open(fn,'r',encoding='utf-8') as f: b = await f.read() while b[0] in ['\n',' ','\t']: b = b[1:] if b.lower().startswith(''): return True if b.lower().startswith(''): return True except Exception as e: return False def url2processor(self, request, url, fpath): config = getConfig() url1 = url url = self.entireUrl(request, url) host = '/'.join(url.split('/')[:3]) path = '/' + '/'.join(url.split('/')[3:]) if config.website.startswiths: for a in config.website.startswiths: if path.startswith(a.leading): processor = FunctionProcessor(path,self,a) return processor if fpath is None: print(f'fpath is None ..., {url=}, {url1=}') return None for word, handlername in self.y_processors: if fpath.endswith(word): Klass = getProcessor(handlername) try: processor = Klass(path,self) # print(f'{f_cnt1=}, {f_cnt2=}, {f_cnt3=}, {f_cnt4=}, {f_cnt5=}') return processor except Exception as e: print('Exception:',e, 'handlername=', handlername) return None return None def websocketUrl(self, request, url): url = entireUrl(request, url) if url.startswith('https'): return 'wss' + url[5:] return 'ws' + url[4:] def entireUrl(self, request, url): if url.startswith('http://') or \ url.startswith('https://') or \ url.startswith('ws://') or \ url.startswith('wss://'): return url if url.startswith('/'): u = f'{self._preurl}{url}' # print(f'entireUrl(), {u=}, {url=}, {self._preurl=}') return u path = request.path p = self.relatedurl(path,url) u = f'{self._preurl}{p}' # print(f'entireUrl(), {u=}, {url=}, {self._preurl=}') return u def url2path(self, url): if url.startswith(self._preurl): return url[len(self._preurl):] return url async def path_call(self, request, path, params={}): url = self.entireUrl(request, path) # print(f'{path=}, after entireUrl(), {url=}') path = self.url2path(url) fpath = self.url2file(path) processor = self.url2processor(request, path, fpath) # print(f'path_call(), {path=}, {url=}, {fpath=}, {processor=}, {self._prepath}') new_request = request.clone(rel_url=path) return await processor.be_call(new_request, params=params)