This commit is contained in:
yumoqing 2025-06-20 21:39:36 +08:00
parent fa20715d63
commit e35fc0545f
10 changed files with 3789 additions and 8 deletions

View File

@ -20,9 +20,25 @@ class BaseEmbedding:
device = torch.device("mps") device = torch.device("mps")
self.model = self.model.to(device) self.model = self.model.to(device)
def embedding(self, doc): def embeddings(self, input):
es = self.model.encode([doc])[0] es = self.model.encode(input)
return es.tolist() data = []
for i, e in enumerate(es):
d = {
"object": "embedding",
"index": i,
"embedding": e.tolist()
}
data.append(d)
return {
"object": "list",
"data": data,
"model": self.model_id.split('/')[-1],
"usage": {
"prompt_tokens": 0,
"total_tokens": 0
}
}
def similarity(self, qvector, dcovectors): def similarity(self, qvector, dcovectors):
s = self.model.similarity([qvector], docvectors) s = self.model.similarity([qvector], docvectors)

View File

@ -16,14 +16,35 @@ from aiohttp_session import get_session
def init(): def init():
rf = RegisterFunction() rf = RegisterFunction()
rf.register('embedding', embedding) rf.register('embeddings', embeddings)
async def embedding(request, params_kw, *params, **kw): async def docs(request, params_kw, *params, **kw):
debug(f'{params_kw.doc=}') txt = """embeddings api:
path: /v1/embeddings
headers: {
"Content-Type": "application/json"
}
data: {
"input": "this is a test"
}
or {
"input":[
"this is first sentence",
"this is second setence"
]
}
async def embeddings(request, params_kw, *params, **kw):
debug(f'{params_kw.input=}')
se = ServerEnv() se = ServerEnv()
engine = se.engine engine = se.engine
f = awaitify(engine.embedding) f = awaitify(engine.embeddings)
arr = await f(params_kw.doc) input = params_kw.input
if input is None:
e = exception(f'input is None')
raise e
if isinstance(input, str):
input = [input]
arr = await f(input)
debug(f'{arr=}, type(arr)') debug(f'{arr=}, type(arr)')
return arr return arr

3
test/chat/Qwen3-0.6B Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
~/models/tsfm.env/bin/python -m llmengine.server -w ~/models/tsfm -p 9999 ~/models/Qwen/Qwen3-0.6B

3
test/chat/Qwen3-Embedding-0.6B Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
~/models/tsfm.env/bin/python -m llmengine.embedding -w ~/models/tsfm -p 9998 ~/models/Qwen/Qwen3-Embedding-0.6B

View File

@ -0,0 +1,51 @@
{
"filesroot":"$[workdir]$/files",
"logger":{
"name":"llmengine",
"levelname":"info",
"logfile":"$[workdir]$/logs/llmengine.log"
},
"website":{
"paths":[
["$[workdir]$/wwwroot",""]
],
"client_max_size":10000,
"host":"0.0.0.0",
"port":9995,
"coding":"utf-8",
"ssl_gg":{
"crtfile":"$[workdir]$/conf/www.bsppo.com.pem",
"keyfile":"$[workdir]$/conf/www.bsppo.com.key"
},
"indexes":[
"index.html",
"index.ui"
],
"startswiths":[
{
"leading":"/idfile",
"registerfunction":"idfile"
},{
"leading": "/v1/chat/completions",
"registerfunction": "chat_completions"
}
],
"processors":[
[".tmpl","tmpl"],
[".app","app"],
[".ui","bui"],
[".dspy","dspy"],
[".md","md"]
],
"rsakey_oops":{
"privatekey":"$[workdir]$/conf/rsa_private_key.pem",
"publickey":"$[workdir]$/conf/rsa_public_key.pem"
},
"session_max_time":3000,
"session_issue_time":2500,
"session_redis_notuse":{
"url":"redis://127.0.0.1:6379"
}
}
}

View File

@ -0,0 +1,6 @@
{
"ymq": {
"ref_text": "\u8f7b\u91cf\u5e94\u7528\u670d\u52a1\u5668\u5907\u6848\u6761\u4ef6\uff1a\u8d2d\u4e70\u65f6\u957f\u57283\u4e2a\u6708\u53ca\u4ee5\u4e0a",
"ref_audio": "/data/ymq/py/f5tts/files/87/103/66/49/record.wav"
}
}

3627
test/chat/logs/llmengine.log Normal file

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,3 @@
#!/bin/bash
~/models/tsfm.env/bin/python -m llmengine.embedding -w ~/models/embedding -p 9998 ~/models/Qwen/Qwen3-Embedding-0.6B

View File

@ -0,0 +1,51 @@
{
"filesroot":"$[workdir]$/files",
"logger":{
"name":"llmengine",
"levelname":"info",
"logfile":"$[workdir]$/logs/llmengine.log"
},
"website":{
"paths":[
["$[workdir]$/wwwroot",""]
],
"client_max_size":10000,
"host":"0.0.0.0",
"port":9995,
"coding":"utf-8",
"ssl_gg":{
"crtfile":"$[workdir]$/conf/www.bsppo.com.pem",
"keyfile":"$[workdir]$/conf/www.bsppo.com.key"
},
"indexes":[
"index.html",
"index.ui"
],
"startswiths":[
{
"leading":"/idfile",
"registerfunction":"idfile"
},{
"leading": "/v1/embeddings",
"registerfunction": "embeddings"
}
],
"processors":[
[".tmpl","tmpl"],
[".app","app"],
[".ui","bui"],
[".dspy","dspy"],
[".md","md"]
],
"rsakey_oops":{
"privatekey":"$[workdir]$/conf/rsa_private_key.pem",
"publickey":"$[workdir]$/conf/rsa_public_key.pem"
},
"session_max_time":3000,
"session_issue_time":2500,
"session_redis_notuse":{
"url":"redis://127.0.0.1:6379"
}
}
}

View File