From fcaee5e657c3a4baac83529ef0d5aa10ef9abd2e Mon Sep 17 00:00:00 2001 From: ymq1 Date: Mon, 23 Jun 2025 04:40:20 +0000 Subject: [PATCH] bugfix --- llmengine/server.py | 8 +- test/embeddings/qwen3-embedding.service | 48 ++++ test/embeddings/start.sh | 3 + test/embeddings/stop.sh | 4 + test/gemma3/.run.sh.swp | Bin 0 -> 12288 bytes test/gemma3/conf/config.json | 51 ++++ test/gemma3/conf/speakers.json | 6 + test/gemma3/gemma3.service | 16 ++ test/gemma3/install.sh | 4 + test/gemma3/logs/llmengine.log | 342 ++++++++++++++++++++++++ test/gemma3/start.sh | 4 + test/gemma3/stop.sh | 3 + test/reranker/conf/config.json | 50 ++++ test/reranker/logs/llmengine.log | 0 test/reranker/qwen3-reranker.service | 48 ++++ test/reranker/start.sh | 4 + test/reranker/stop.sh | 5 + test/reranker/t.sh | 17 ++ 18 files changed, 609 insertions(+), 4 deletions(-) create mode 100644 test/embeddings/qwen3-embedding.service create mode 100755 test/embeddings/start.sh create mode 100755 test/embeddings/stop.sh create mode 100644 test/gemma3/.run.sh.swp create mode 100644 test/gemma3/conf/config.json create mode 100644 test/gemma3/conf/speakers.json create mode 100644 test/gemma3/gemma3.service create mode 100755 test/gemma3/install.sh create mode 100644 test/gemma3/logs/llmengine.log create mode 100755 test/gemma3/start.sh create mode 100644 test/gemma3/stop.sh create mode 100644 test/reranker/conf/config.json create mode 100644 test/reranker/logs/llmengine.log create mode 100644 test/reranker/qwen3-reranker.service create mode 100755 test/reranker/start.sh create mode 100755 test/reranker/stop.sh create mode 100755 test/reranker/t.sh diff --git a/llmengine/server.py b/llmengine/server.py index 0220090..61b5a5f 100644 --- a/llmengine/server.py +++ b/llmengine/server.py @@ -3,10 +3,10 @@ import os import sys import argparse -from base_chat_llm import BaseChatLLM, get_llm_class -from gemma3_it import Gemma3LLM -from medgemma3_it import MedgemmaLLM -from qwen3 import Qwen3LLM +from llmengine.base_chat_llm import BaseChatLLM, get_llm_class +from llmengine.gemma3_it import Gemma3LLM +from llmengine.medgemma3_it import MedgemmaLLM +from llmengine.qwen3 import Qwen3LLM from appPublic.registerfunction import RegisterFunction from appPublic.log import debug, exception diff --git a/test/embeddings/qwen3-embedding.service b/test/embeddings/qwen3-embedding.service new file mode 100644 index 0000000..c2bd2d4 --- /dev/null +++ b/test/embeddings/qwen3-embedding.service @@ -0,0 +1,48 @@ +[Unit] +Description=A Embedding Service using Qwen3-Embedding-0.6B +# After=network.target DeepSeek70B-kyyds671b-ray.service +# Requires=DeepSeek70B-kyyds671b-ray.service +StartLimitIntervalSec=60 +StartLimitBurst=5 + +[Service] +# 核心启动参数(保持原有配置) +User=ymq +Group=ymq +WorkingDirectory=/share/ymq/run/embeddings +#定义环境变量,所有节点的启动脚本与服务需一致 +#Environment="NCCL_SOCKET_IFNAME=enp196s0f0np0" +#ExecStartPre=/data/kyyds671b/ray_check.sh +ExecStart=/share/ymq/run/embeddings/start.sh +ExecStop=/share/ymq/run/embeddings/stop.sh + +# 超时与停止控制(新增部分) +# 启动超时延长至 120 秒 +# TimeoutStartSec=120 +# 停止等待时间 30 秒 +# TimeoutStopSec=30 +# 优先发送 SIGINT 信号(更适合 Python 程序) +# KillSignal=SIGINT +# 最终强制终止信号 +# RestartKillSignal=SIGKILL +# 混合终止模式 +# KillMode=mixed +# 重启策略 +# Restart=on-failure +# RestartSec=10s +# 服务管理(保持原有配置+增强) +#Restart=always +#RestartSec=10 # 重启间隔从 5 秒调整为 10 秒 +#append 是继续写入相当于>> file是从新写入 相当于> +StandardOutput=append:/var/log/embeddings/embeddings.log +StandardError=append:/var/log/embeddings/error.log +SyslogIdentifier=embeddings +# 资源限制(保持可选配置) +#LimitNOFILE=65536 +#LimitNPROC=65536 +# GPU 支持 +#Environment=CUDA_VISIBLE_DEVICES=0,1 + +[Install] +WantedBy=multi-user.target + diff --git a/test/embeddings/start.sh b/test/embeddings/start.sh new file mode 100755 index 0000000..343630b --- /dev/null +++ b/test/embeddings/start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +CUDA_VISIBLE_DEVICES=7 /share/vllm-0.8.5/bin/python -m llmengine.embedding -p 9998 /d/ymq/models/Qwen/Qwen3-Embedding-0.6B diff --git a/test/embeddings/stop.sh b/test/embeddings/stop.sh new file mode 100755 index 0000000..470ec8f --- /dev/null +++ b/test/embeddings/stop.sh @@ -0,0 +1,4 @@ +#!/usr/bin/bash + +killname Qwen/Qwen3-Embedding + diff --git a/test/gemma3/.run.sh.swp b/test/gemma3/.run.sh.swp new file mode 100644 index 0000000000000000000000000000000000000000..944594dfa73d52ef244c8a7a59930a6941372588 GIT binary patch literal 12288 zcmeI&O-jQ+6bJCBt`tO34=~kjCo@g_NEa^bhY-+3De1N(bZiRwawe^InKo?o3UKYQa+JM5;*0frK5E{s6zk(8xWWkkIX&r9lI&YY3FBk zdUEVz{}fB*y_009U< z00Izz00bbgg#y8tkp3Yd3v-qK|5xAt-)8+))!)i@2nav`0uX=z1Rwwb2tWV=5P$## z{x8t+ySJmEH}Kl`y-~L}^g6x4jc7C*cEh37N%?skCrntbb;-kN%JW4LWhrF|HFcRz vrm3{FRCB2)%QIEbmWfqwl8t4odH-3a%bUX-yKT<>*7;1U<+v~~$L@as9p*{% literal 0 HcmV?d00001 diff --git a/test/gemma3/conf/config.json b/test/gemma3/conf/config.json new file mode 100644 index 0000000..692c878 --- /dev/null +++ b/test/gemma3/conf/config.json @@ -0,0 +1,51 @@ +{ + "filesroot":"$[workdir]$/files", + "logger":{ + "name":"llmengine", + "levelname":"info", + "logfile":"$[workdir]$/logs/llmengine.log" + }, + "website":{ + "paths":[ + ["$[workdir]$/wwwroot",""] + ], + "client_max_size":10000, + "host":"0.0.0.0", + "port":9995, + "coding":"utf-8", + "ssl_gg":{ + "crtfile":"$[workdir]$/conf/www.bsppo.com.pem", + "keyfile":"$[workdir]$/conf/www.bsppo.com.key" + }, + "indexes":[ + "index.html", + "index.ui" + ], + "startswiths":[ + { + "leading":"/idfile", + "registerfunction":"idfile" + },{ + "leading": "/v1/chat/completions", + "registerfunction": "chat_completions" + } + ], + "processors":[ + [".tmpl","tmpl"], + [".app","app"], + [".ui","bui"], + [".dspy","dspy"], + [".md","md"] + ], + "rsakey_oops":{ + "privatekey":"$[workdir]$/conf/rsa_private_key.pem", + "publickey":"$[workdir]$/conf/rsa_public_key.pem" + }, + "session_max_time":3000, + "session_issue_time":2500, + "session_redis_notuse":{ + "url":"redis://127.0.0.1:6379" + } + } +} + diff --git a/test/gemma3/conf/speakers.json b/test/gemma3/conf/speakers.json new file mode 100644 index 0000000..9b0ea52 --- /dev/null +++ b/test/gemma3/conf/speakers.json @@ -0,0 +1,6 @@ +{ + "ymq": { + "ref_text": "\u8f7b\u91cf\u5e94\u7528\u670d\u52a1\u5668\u5907\u6848\u6761\u4ef6\uff1a\u8d2d\u4e70\u65f6\u957f\u57283\u4e2a\u6708\u53ca\u4ee5\u4e0a", + "ref_audio": "/data/ymq/py/f5tts/files/87/103/66/49/record.wav" + } +} diff --git a/test/gemma3/gemma3.service b/test/gemma3/gemma3.service new file mode 100644 index 0000000..e11c81b --- /dev/null +++ b/test/gemma3/gemma3.service @@ -0,0 +1,16 @@ +[Unit] +Wants=systemd-networkd.service + +[Service] +User=ymq +Group=ymq +Type=forking +WorkingDirectory=/share/ymq/run/gemma3 +ExecStart=/share/ymq/run/gemma3/start.sh +ExecStop=/share/ymq/run/gemma3/stop.sh +StandardOutput=append:/var/log/gemma3/gemma3.log +StandardError=append:/var/log/gemma3/gemma3.log +SyslogIdentifier=gemma3 + +[Install] +WantedBy=multi-user.target diff --git a/test/gemma3/install.sh b/test/gemma3/install.sh new file mode 100755 index 0000000..b562479 --- /dev/null +++ b/test/gemma3/install.sh @@ -0,0 +1,4 @@ +sudo mkdir /var/log/gemma3 +sudo cp gemma3.service /etc/systemd/system +sudo systemctl enable gemma3 +sudo systemctl start gemma3 diff --git a/test/gemma3/logs/llmengine.log b/test/gemma3/logs/llmengine.log new file mode 100644 index 0000000..7f34068 --- /dev/null +++ b/test/gemma3/logs/llmengine.log @@ -0,0 +1,342 @@ +2025-06-09 08:13:26.400[llmengine][info][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:151]checkAuth() called ... request.path='/v1/chat/completions' +2025-06-09 08:13:26.411[llmengine][debug][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py:40]params_kw={'{\n"prompt":"who are you"\n}': ''}, args=[] +2025-06-09 08:13:26.418[llmengine][exception][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:168]Exception=client(127.0.0.1) None access /v1/chat/completions cost 0.008466005325317383, (0.000392913818359375), except=name 'stream_response' is not defined +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 42, in chat_completions + return await stream_response(request, gor) +NameError: name 'stream_response' is not defined + +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 42, in chat_completions + return await stream_response(request, gor) +NameError: name 'stream_response' is not defined + +2025-06-09 08:15:08.876[llmengine][info][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:151]checkAuth() called ... request.path='/v1/chat/completions' +2025-06-09 08:15:08.884[llmengine][debug][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py:40]params_kw={'{\n"prompt":"who are you"\n}': ''}, args=[] +2025-06-09 08:15:08.891[llmengine][exception][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:168]Exception=client(127.0.0.1) None access /v1/chat/completions cost 0.005657672882080078, (9.679794311523438e-05), except=get_session() missing 1 required positional argument: 'request' +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 43, in chat_completions + return await stream_response(request, gor) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/globalEnv.py", line 58, in stream_response + async for d in async_data_generator(): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 31, in gor + session = await get_session() +TypeError: get_session() missing 1 required positional argument: 'request' + +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 43, in chat_completions + return await stream_response(request, gor) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/globalEnv.py", line 58, in stream_response + async for d in async_data_generator(): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 31, in gor + session = await get_session() +TypeError: get_session() missing 1 required positional argument: 'request' + +2025-06-09 08:19:30.169[llmengine][info][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:151]checkAuth() called ... request.path='/v1/chat/completions' +2025-06-09 08:19:30.177[llmengine][debug][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py:40]params_kw={'{\n"prompt":"who are you"\n}': ''}, args=[] +2025-06-09 08:19:30.223[llmengine][exception][/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py:168]Exception=client(127.0.0.1) None access /v1/chat/completions cost 0.03934144973754883, (0.00010514259338378906), except='None' has no attribute 'startswith' +Traceback (most recent call last): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/auth_api.py", line 161, in checkAuth + ret = await handler(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/processorResource.py", line 351, in _handle + ret = await processor.handle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 95, in handle + await self.execute(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/baseProcessor.py", line 86, in execute + await self.datahandle(request) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 46, in datahandle + x = await self.path_call(request, self.path) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/functionProcessor.py", line 42, in path_call + return await f(request, params_kw, *args) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 45, in chat_completions + return await stream_response(request, gor) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/ahserver/globalEnv.py", line 58, in stream_response + async for d in async_data_generator(): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/server.py", line 41, in gor + async for d in engine.async_stream_generate(session, params_kw.prompt, **kwargs): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/base_chat_llm.py", line 112, in async_stream_generate + for d in self._generator(session, prompt, + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/base_chat_llm.py", line 66, in _generator + for d in self._gen(messages): + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/base_chat_llm.py", line 137, in _gen + inputs = self._messages2inputs(messages) + File "/share/vllm-0.8.5/lib/python3.10/site-packages/llmengine/qwen3.py", line 32, in _messages2inputs + text = self.tokenizer.apply_chat_template( + File "/share/vllm-0.8.5/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 1695, in apply_chat_template + rendered_chat = compiled_template.render( + File "/share/vllm-0.8.5/lib/python3.10/site-packages/jinja2/environment.py", line 1295, in render + self.environment.handle_exception() + File "/share/vllm-0.8.5/lib/python3.10/site-packages/jinja2/environment.py", line 942, in handle_exception + raise rewrite_traceback_stack(source=source) + File "