From a9eb8c1a3056dc50179cc5f5631e9372906b6313 Mon Sep 17 00:00:00 2001 From: ymq1 Date: Wed, 25 Jun 2025 09:21:45 +0800 Subject: [PATCH] bugfix --- llmengine/base_chat_llm.py | 2 +- llmengine/client/m2t | 74 ++++++++++++++++++++++++++++++++++++++ llmengine/client/t2t | 67 ++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100755 llmengine/client/m2t create mode 100755 llmengine/client/t2t diff --git a/llmengine/base_chat_llm.py b/llmengine/base_chat_llm.py index c4634e8..cb03117 100644 --- a/llmengine/base_chat_llm.py +++ b/llmengine/base_chat_llm.py @@ -49,7 +49,7 @@ class BaseChatLLM: all_txt = '' t1 = time() i = 0 - id = f'chatllm-{getID}' + id = f'chatllm-{getID()}' for txt in streamer: if txt == '': continue diff --git a/llmengine/client/m2t b/llmengine/client/m2t new file mode 100755 index 0000000..a06becd --- /dev/null +++ b/llmengine/client/m2t @@ -0,0 +1,74 @@ +#!/d/ymq/py3/bin/python +from traceback import format_exc +import asyncio +import codecs +import json +import argparse +from appPublic.streamhttpclient import liner, StreamHttpClient +from appPublic.log import MyLogger + +def system_message(prompt): + return { + 'role':'system', + 'content':[{ + 'type': 'text', + 'text': prompt + }] + } + +def user_message(prompt, filepath=None, **kwargs): + if filepath: + prompt += f':{user_file(filepath)}' + content = [{ + 'type': 'text', + 'text': prompt + }] + return { + 'role': 'user', + 'content': content + } + +def user_file(fn): + with codecs.open(fn, 'r', 'utf-8') as f: + return f.read() + +async def main(): + parser = argparse.ArgumentParser(prog='devops') + parser.add_argument('-f', '--file') + parser.add_argument('-p', '--prompt') + parser.add_argument('-s', '--sys_prompt') + parser.add_argument('-m', '--model') + parser.add_argument('url') + args = parser.parse_args() + messages = [ system_message(args.sys_prompt) ] if args.sys_prompt else [] + messages.append(user_message(args.prompt, filepath=args.file)) + + d = { + 'model': args.model, + 'stream': True, + 'messages': messages + } + hc = StreamHttpClient() + headers = { + 'Content-Type': 'application/json' + } + i = 0 + buffer = '' + reco = hc('POST', args.url, headers=headers, data=json.dumps(d)) + async for chunk in liner(reco): + chunk = chunk[6:] + if chunk != '[DONE]': + try: + f = json.loads(chunk) + except Exception as e: + print(f'****{chunk=} error {e} {format_exc()}') + continue + if not f['choices'][0]['finish_reason']: + print(f['choices'][0]['delta']['content'], end='', flush=True) + else: + pass + print('\n\n') + +if __name__ == '__main__': + MyLogger('null', levelname='error', logfile='/dev/null') + asyncio.new_event_loop().run_until_complete(main()) diff --git a/llmengine/client/t2t b/llmengine/client/t2t new file mode 100755 index 0000000..69e6093 --- /dev/null +++ b/llmengine/client/t2t @@ -0,0 +1,67 @@ +#!/d/ymq/py3/bin/python +from traceback import format_exc +import asyncio +import codecs +import json +import argparse +from appPublic.streamhttpclient import liner, StreamHttpClient +from appPublic.log import MyLogger + +def system_message(prompt): + return { + 'role':'system', + 'content': prompt + } + +def user_message(prompt, filepath=None): + if filepath: + prompt += f':{user_file(filepath)}' + return { + 'role': 'user', + 'content': prompt + } + +def user_file(fn): + with codecs.open(fn, 'r', 'utf-8') as f: + return f.read() + +async def main(): + parser = argparse.ArgumentParser(prog='devops') + parser.add_argument('-f', '--file') + parser.add_argument('-p', '--prompt') + parser.add_argument('-s', '--sys_prompt') + parser.add_argument('-m', '--model') + parser.add_argument('url') + args = parser.parse_args() + messages = [ system_message(args.sys_prompt) ] if args.sys_prompt else [] + messages.append(user_message(args.prompt, filepath=args.file)) + + d = { + 'model': args.model, + 'stream': True, + 'messages': messages + } + hc = StreamHttpClient() + headers = { + 'Content-Type': 'application/json' + } + i = 0 + buffer = '' + reco = hc('POST', args.url, headers=headers, data=json.dumps(d)) + async for chunk in liner(reco): + chunk = chunk[6:] + if chunk != '[DONE]': + try: + f = json.loads(chunk) + except Exception as e: + print(f'****{chunk=} error {e} {format_exc()}') + continue + if not f['choices'][0]['finish_reason']: + print(f['choices'][0]['delta']['content'], end='', flush=True) + else: + pass + print('\n\n') + +if __name__ == '__main__': + MyLogger('null', levelname='error', logfile='/dev/null') + asyncio.new_event_loop().run_until_complete(main())