From ca11f86ee4a5e42fd1ca3ee7095bf65eb847d4a6 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Mon, 9 Jun 2025 07:04:30 +0000 Subject: [PATCH] bugfix --- llmengine/server.py | 57 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 llmengine/server.py diff --git a/llmengine/server.py b/llmengine/server.py new file mode 100644 index 0000000..c29d625 --- /dev/null +++ b/llmengine/server.py @@ -0,0 +1,57 @@ +from traceback import format_exc +import sys +import argsparse +from llmengine.gemma3_it import Gemma3ChatLLM +from llmengine.qwen3 import Qwen3ChatLLM +from llmengine.medgemma3_it import Medgemma3ChatLLM +from ahserver.webapp import server +from from aiohttp_session import get_session +model_pathMap = { + "/share/models/Qwen/Qwen3-32B": Qwen3ChatLLM, + "/share/models/google/gemma-3-4b-it": Gemma3ChatLLM, + "/share/models/google/medgemma-4b-it": Medgemma3ChatLLM +} +def register(model_path, Klass): + model_pathMap[model_path] = Klass + +def init(self): + rf = RegisterFunction() + rf.register('chat_completions', chat_completions) + +async def chat_completions(request, params_kw, *params, **kw): + async def gor(): + se = ServerEnv() + engine = se.chat_engine + session = await get_session() + kwargs = { + } + if params_kw.image_path: + kwargs['image_path'] = fs.reapPath(params_kw.image_path) + if params_kw.video_path: + kwargs['video_path'] = fs.reapPath(params_kw.video_path) + if params_kw.audio_path: + kwargs['audio_path'] = fs.reapPath(params_kw.audio_path) + async for d in engine.async_stream_generate(session, params_kw.prompt, **kwargs): + yield d + + return await stream_response(request, gor) + +def main(): + parser = argparse.ArgumentParser(prog="Sage") + parser.add_argument('-w', '--workdir') + parser.add_argument('-p', '--port') + parser.add_argument('model_path') + args = parser.parse_args() + Klass = model_pathMap.get(args.model_path) + if Klass is None: + e = Exception(f'{model_path} has not mapping to a model class') + exception(f'{e}, {format_exc()}') + raise e + se.chat_engine = Klass(args.model_path) + workdir = args.workdir or os.getcwd() + port = args.port + server(init_func, workdir, port) + +if __name__ == '__main__': + main() +