diff --git a/llmengine/server.py b/llmengine/server.py index 0220090..61b5a5f 100644 --- a/llmengine/server.py +++ b/llmengine/server.py @@ -3,10 +3,10 @@ import os import sys import argparse -from base_chat_llm import BaseChatLLM, get_llm_class -from gemma3_it import Gemma3LLM -from medgemma3_it import MedgemmaLLM -from qwen3 import Qwen3LLM +from llmengine.base_chat_llm import BaseChatLLM, get_llm_class +from llmengine.gemma3_it import Gemma3LLM +from llmengine.medgemma3_it import MedgemmaLLM +from llmengine.qwen3 import Qwen3LLM from appPublic.registerfunction import RegisterFunction from appPublic.log import debug, exception diff --git a/test/embeddings/qwen3-embedding.service b/test/embeddings/qwen3-embedding.service new file mode 100644 index 0000000..c2bd2d4 --- /dev/null +++ b/test/embeddings/qwen3-embedding.service @@ -0,0 +1,48 @@ +[Unit] +Description=A Embedding Service using Qwen3-Embedding-0.6B +# After=network.target DeepSeek70B-kyyds671b-ray.service +# Requires=DeepSeek70B-kyyds671b-ray.service +StartLimitIntervalSec=60 +StartLimitBurst=5 + +[Service] +# 核心启动参数(保持原有配置) +User=ymq +Group=ymq +WorkingDirectory=/share/ymq/run/embeddings +#定义环境变量,所有节点的启动脚本与服务需一致 +#Environment="NCCL_SOCKET_IFNAME=enp196s0f0np0" +#ExecStartPre=/data/kyyds671b/ray_check.sh +ExecStart=/share/ymq/run/embeddings/start.sh +ExecStop=/share/ymq/run/embeddings/stop.sh + +# 超时与停止控制(新增部分) +# 启动超时延长至 120 秒 +# TimeoutStartSec=120 +# 停止等待时间 30 秒 +# TimeoutStopSec=30 +# 优先发送 SIGINT 信号(更适合 Python 程序) +# KillSignal=SIGINT +# 最终强制终止信号 +# RestartKillSignal=SIGKILL +# 混合终止模式 +# KillMode=mixed +# 重启策略 +# Restart=on-failure +# RestartSec=10s +# 服务管理(保持原有配置+增强) +#Restart=always +#RestartSec=10 # 重启间隔从 5 秒调整为 10 秒 +#append 是继续写入相当于>> file是从新写入 相当于> +StandardOutput=append:/var/log/embeddings/embeddings.log +StandardError=append:/var/log/embeddings/error.log +SyslogIdentifier=embeddings +# 资源限制(保持可选配置) +#LimitNOFILE=65536 +#LimitNPROC=65536 +# GPU 支持 +#Environment=CUDA_VISIBLE_DEVICES=0,1 + +[Install] +WantedBy=multi-user.target + diff --git a/test/embeddings/start.sh b/test/embeddings/start.sh new file mode 100755 index 0000000..343630b --- /dev/null +++ b/test/embeddings/start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +CUDA_VISIBLE_DEVICES=7 /share/vllm-0.8.5/bin/python -m llmengine.embedding -p 9998 /d/ymq/models/Qwen/Qwen3-Embedding-0.6B diff --git a/test/embeddings/stop.sh b/test/embeddings/stop.sh new file mode 100755 index 0000000..470ec8f --- /dev/null +++ b/test/embeddings/stop.sh @@ -0,0 +1,4 @@ +#!/usr/bin/bash + +killname Qwen/Qwen3-Embedding + diff --git a/test/gemma3/.run.sh.swp b/test/gemma3/.run.sh.swp new file mode 100644 index 0000000..944594d Binary files /dev/null and b/test/gemma3/.run.sh.swp differ diff --git a/test/gemma3/conf/config.json b/test/gemma3/conf/config.json new file mode 100644 index 0000000..692c878 --- /dev/null +++ b/test/gemma3/conf/config.json @@ -0,0 +1,51 @@ +{ + "filesroot":"$[workdir]$/files", + "logger":{ + "name":"llmengine", + "levelname":"info", + "logfile":"$[workdir]$/logs/llmengine.log" + }, + "website":{ + "paths":[ + ["$[workdir]$/wwwroot",""] + ], + "client_max_size":10000, + "host":"0.0.0.0", + "port":9995, + "coding":"utf-8", + "ssl_gg":{ + "crtfile":"$[workdir]$/conf/www.bsppo.com.pem", + "keyfile":"$[workdir]$/conf/www.bsppo.com.key" + }, + "indexes":[ + "index.html", + "index.ui" + ], + "startswiths":[ + { + "leading":"/idfile", + "registerfunction":"idfile" + },{ + "leading": "/v1/chat/completions", + "registerfunction": "chat_completions" + } + ], + "processors":[ + [".tmpl","tmpl"], + [".app","app"], + [".ui","bui"], + [".dspy","dspy"], + [".md","md"] + ], + "rsakey_oops":{ + "privatekey":"$[workdir]$/conf/rsa_private_key.pem", + "publickey":"$[workdir]$/conf/rsa_public_key.pem" + }, + "session_max_time":3000, + "session_issue_time":2500, + "session_redis_notuse":{ + "url":"redis://127.0.0.1:6379" + } + } +} + diff --git a/test/gemma3/conf/speakers.json b/test/gemma3/conf/speakers.json new file mode 100644 index 0000000..9b0ea52 --- /dev/null +++ b/test/gemma3/conf/speakers.json @@ -0,0 +1,6 @@ +{ + "ymq": { + "ref_text": "\u8f7b\u91cf\u5e94\u7528\u670d\u52a1\u5668\u5907\u6848\u6761\u4ef6\uff1a\u8d2d\u4e70\u65f6\u957f\u57283\u4e2a\u6708\u53ca\u4ee5\u4e0a", + "ref_audio": "/data/ymq/py/f5tts/files/87/103/66/49/record.wav" + } +} diff --git a/test/gemma3/gemma3.service b/test/gemma3/gemma3.service new file mode 100644 index 0000000..e11c81b --- /dev/null +++ b/test/gemma3/gemma3.service @@ -0,0 +1,16 @@ +[Unit] +Wants=systemd-networkd.service + +[Service] +User=ymq +Group=ymq +Type=forking +WorkingDirectory=/share/ymq/run/gemma3 +ExecStart=/share/ymq/run/gemma3/start.sh +ExecStop=/share/ymq/run/gemma3/stop.sh +StandardOutput=append:/var/log/gemma3/gemma3.log +StandardError=append:/var/log/gemma3/gemma3.log +SyslogIdentifier=gemma3 + +[Install] +WantedBy=multi-user.target diff --git a/test/gemma3/install.sh b/test/gemma3/install.sh new file mode 100755 index 0000000..b562479 --- /dev/null +++ b/test/gemma3/install.sh @@ -0,0 +1,4 @@ +sudo mkdir /var/log/gemma3 +sudo cp gemma3.service /etc/systemd/system +sudo systemctl enable gemma3 +sudo systemctl start gemma3 diff --git a/test/gemma3/logs/llmengine.log b/test/gemma3/logs/llmengine.log new file mode 100644 index 0000000..7f34068 --- /dev/null +++ b/test/gemma3/logs/llmengine.log @@ -0,0 +1,342 @@ +2025-06-09 08:13:26.400[llmengine][info][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:151]checkAuth() called ... request.path='/v1/chat/completions' +2025-06-09 08:13:26.411[llmengine][debug][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py:40]params_kw={'{\n"prompt":"who are you"\n}': ''}, args=[] +2025-06-09 08:13:26.418[llmengine][exception][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:168]Exception=client(127.0.0.1) None access /v1/chat/completions cost 0.008466005325317383, (0.000392913818359375), except=name 'stream_response' is not defined +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 42, in chat_completions + return await stream_response(request, gor) +NameError: name 'stream_response' is not defined + +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 42, in chat_completions + return await stream_response(request, gor) +NameError: name 'stream_response' is not defined + +2025-06-09 08:15:08.876[llmengine][info][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:151]checkAuth() called ... request.path='/v1/chat/completions' +2025-06-09 08:15:08.884[llmengine][debug][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py:40]params_kw={'{\n"prompt":"who are you"\n}': ''}, args=[] +2025-06-09 08:15:08.891[llmengine][exception][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:168]Exception=client(127.0.0.1) None access /v1/chat/completions cost 0.005657672882080078, (9.679794311523438e-05), except=get_session() missing 1 required positional argument: 'request' +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 43, in chat_completions + return await stream_response(request, gor) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/globalEnv.py", line 58, in stream_response + async for d in async_data_generator(): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 31, in gor + session = await get_session() +TypeError: get_session() missing 1 required positional argument: 'request' + +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 43, in chat_completions + return await stream_response(request, gor) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/globalEnv.py", line 58, in stream_response + async for d in async_data_generator(): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 31, in gor + session = await get_session() +TypeError: get_session() missing 1 required positional argument: 'request' + +2025-06-09 08:19:30.169[llmengine][info][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:151]checkAuth() called ... request.path='/v1/chat/completions' +2025-06-09 08:19:30.177[llmengine][debug][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py:40]params_kw={'{\n"prompt":"who are you"\n}': ''}, args=[] +2025-06-09 08:19:30.223[llmengine][exception][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:168]Exception=client(127.0.0.1) None access /v1/chat/completions cost 0.03934144973754883, (0.00010514259338378906), except='None' has no attribute 'startswith' +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 45, in chat_completions + return await stream_response(request, gor) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/globalEnv.py", line 58, in stream_response + async for d in async_data_generator(): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 41, in gor + async for d in engine.async_stream_generate(session, params_kw.prompt, **kwargs): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/base_chat_llm.py", line 112, in async_stream_generate + for d in self._generator(session, prompt, + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/base_chat_llm.py", line 66, in _generator + for d in self._gen(messages): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/base_chat_llm.py", line 137, in _gen + inputs = self._messages2inputs(messages) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/qwen3.py", line 32, in _messages2inputs + text = self.tokenizer.apply_chat_template( + File "/share/vllm-0.8.5/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 1695, in apply_chat_template + rendered_chat = compiled_template.render( + File "/share/vllm-0.8.5/lib/python3.10/site-packages/jinja2/environment.py", line 1295, in render + self.environment.handle_exception() + File "/share/vllm-0.8.5/lib/python3.10/site-packages/jinja2/environment.py", line 942, in handle_exception + raise rewrite_traceback_stack(source=source) + File "