From 70ce7a480108e046645de47cec39f43bf9ee524a Mon Sep 17 00:00:00 2001
From: yumoqing <yumoqing@gmail.com>
Date: Tue, 12 Nov 2024 15:32:14 +0800
Subject: [PATCH] bugficx

---
 README.md              |  41 +++++++++++++++++
 app/main.py            | 101 +++++++++++++++++++++++++++++++++++++++++
 conf/config.json       |  47 +++++++++++++++++++
 requirements.txt       |   2 +
 script/install.sh      |   3 ++
 script/qwenvl.service  |  13 ++++++
 script/qwenvl.sh       |   5 ++
 wwwroot/api/index.dspy |   3 ++
 8 files changed, 215 insertions(+)
 create mode 100644 app/main.py
 create mode 100755 conf/config.json
 create mode 100644 requirements.txt
 create mode 100644 script/install.sh
 create mode 100644 script/qwenvl.service
 create mode 100755 script/qwenvl.sh
 create mode 100644 wwwroot/api/index.dspy

diff --git a/README.md b/README.md
index e69de29..c654954 100644
--- a/README.md
+++ b/README.md
@@ -0,0 +1,41 @@
+# Qwen2-VL deployment instances
+
+# dependents
+git+https://git.kaiyuancloud.cn/yumoqing/apppublic
+git+https://git.kaiyuancloud.cn/yumoqing/ahserver
+
+# preinstallation
+first, create a new python virtual env
+```
+python3 -m venv ~/vl
+```
+create two shell scripts named vlpy and vlpip:
+```vlpy
+#!/usr/bin/bash
+~/vl/bin/python $*
+```
+and 
+```vlpip
+#!/usr/bin/bash
+~/vl/bin/pip $*
+```
+and copy them to the bin under you $HOME folder, and chmod +x to them
+```
+mv vlpip vlpy ~/bin
+chmod +x ~/bin/vl*
+```
+
+follow instuctions from [Qwen2-VL](https://github.com/QwenLM/Qwen2-VL), remember to change pip to vlpip
+
+## isntallation
+
+do the following
+```
+git clone https://git.kaiyauncloud.cn/yumoqing/qwenvl
+cd qwenvl/script
+sudo isntall.sh
+```
+
+## Change model or http port
+there is a config.json file under qwenvl folder, change the "modelname" and "port" value to suite your requirements
+
diff --git a/app/main.py b/app/main.py
new file mode 100644
index 0000000..eea1e66
--- /dev/null
+++ b/app/main.py
@@ -0,0 +1,101 @@
+import torch
+from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor
+from qwen_vl_utils import process_vision_info
+from appPublic.worker import awaitify
+from appPublic.jsonConfig import getConfig
+from ahserver.serverenv import ServerEnv
+from ahserver.webapp import webapp
+
+class Qwen2VLClass:
+	def __init__(self, modelname):
+		# default: Load the model on the available device(s)
+		self.model = Qwen2VLForConditionalGeneration.from_pretrained(
+			modelname,
+			torch_dtype=torch.bfloat16,
+			# attn_implementation="flash_attention_2",
+			device_map="auto"
+		)
+
+		# We recommend enabling flash_attention_2 for better acceleration and memory saving, especially in multi-image and video scenarios.
+		# model = Qwen2VLForConditionalGeneration.from_pretrained(
+		#     "Qwen/Qwen2-VL-7B-Instruct",
+		#     torch_dtype=torch.bfloat16,
+		#     attn_implementation="flash_attention_2",
+		#     device_map="auto",
+		# )
+
+		# default processer
+		self.processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct")
+
+		# The default range for the number of visual tokens per image in the model is 4-16384.
+		# You can set min_pixels and max_pixels according to your needs, such as a token range of 256-1280, to balance performance and cost.
+		# min_pixels = 256*28*28
+		# max_pixels = 1280*28*28
+		# processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct", min_pixels=min_pixels, max_pixels=max_pixels)
+
+	def inference(self, prompt, image=None, videofile=None):
+		content = [
+			{
+				"type":"text",
+				"text":prompt
+			}
+		]
+		if image:
+			if not image.startswith('file:///') \
+						and not image.startswith('http://') \
+						and not image.startswith('https://'):
+				image = f'data:image;base64,{image}'
+			content.append({
+				"type":"image",
+				"image":image
+			})
+		if videofile:
+			if not videofile.startswith('file:///'):
+				return 'only local video file support'
+
+			content.append({
+				"type":"video",
+				"video":videofile
+			})
+		
+		messages = [
+			{
+				"role": "user",
+				"content": content
+			}
+		]
+
+		# Preparation for inference
+		text = self.processor.apply_chat_template(
+			messages, tokenize=False, add_generation_prompt=True
+		)
+		image_inputs, video_inputs = process_vision_info(messages)
+		inputs = self.processor(
+			text=[text],
+			images=image_inputs,
+			videos=video_inputs,
+			padding=True,
+			return_tensors="pt",
+		)
+		inputs = inputs.to("cuda")
+
+		# Inference: Generation of the output
+		generated_ids = self.model.generate(**inputs, max_new_tokens=128)
+		generated_ids_trimmed = [
+			out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+		]
+		output_text = processor.batch_decode(
+			generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+		)
+		return output_text
+
+def main():
+	config = getConfig()
+	modelname = config.modelname
+	m = Qwen2VLClass(modelname)
+	g = ServerEnv()
+	g.inference = awaitify(m.inference)
+
+
+if __name__ == '__main__':
+	webapp(main)
diff --git a/conf/config.json b/conf/config.json
new file mode 100755
index 0000000..684fd84
--- /dev/null
+++ b/conf/config.json
@@ -0,0 +1,47 @@
+{
+	"password_key":"!@#$%^&*(*&^%$QWERTYUIqwertyui234567",
+	"modelname":"Qwen/Qwen2-VL-7B-Instruct",
+	"logger":{
+		"name":"qwenvl",
+		"levelname":"info",
+		"logfile":"$[workdir]$/logs/sage.log"
+	},
+	"filesroot":"$[workdir]$/files",
+	"website":{
+		"paths":[
+			["$[workdir]$/wwwroot",""]
+		],
+		"client_max_size":10000,
+		"host":"0.0.0.0",
+		"port":10090,
+		"coding":"utf-8",
+		"indexes":[
+			"index.html",
+			"index.tmpl",
+			"index.ui",
+			"index.dspy",
+			"index.md"
+		],
+		"startswiths":[
+			{
+				"leading":"/idfile",
+				"registerfunction":"idFileDownload"
+			}
+		],
+		"processors":[
+			[".dspy","dspy"],
+			[".md","md"]
+		],
+		"session_max_time":3000,
+		"session_issue_time":2500,
+		"session_redis_notuse":{
+			"url":"redis://127.0.0.1:6379"
+		}
+	},
+	"langMapping":{
+		"zh-Hans-CN":"zh-cn",
+		"zh-CN":"zh-cn",
+		"en-us":"en",
+		"en-US":"en"
+	}
+}
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..b139459
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,2 @@
+git+https://git.kaiyuancloud.cn/yumoqing/apppublic
+git+https://git.kaiyuancloud.cn/yumoqing/ahserver
diff --git a/script/install.sh b/script/install.sh
new file mode 100644
index 0000000..bbd75cd
--- /dev/null
+++ b/script/install.sh
@@ -0,0 +1,3 @@
+sudo cp qwenvl.service /etc/systemd/system
+sudo systemctl enable qwenvl.service
+sudo systemctl start qwenvl
diff --git a/script/qwenvl.service b/script/qwenvl.service
new file mode 100644
index 0000000..cd4d304
--- /dev/null
+++ b/script/qwenvl.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=qwen2-vl inference service
+Documention=qwen2-vl inference service to control sage service start or stop
+Wants=systemd-networkd.service
+Requires=nginx.service
+
+[Service]
+Type=forking
+ExecStart=su - ymq -c "/d/ymq/py/qwenvl/script/qwenvl.sh"
+ExecStop=su - ymq "/d/ymq/bin/killname qwenvl.py"
+[Install]
+WantedBy=multi-user.target
+
diff --git a/script/qwenvl.sh b/script/qwenvl.sh
new file mode 100755
index 0000000..133c934
--- /dev/null
+++ b/script/qwenvl.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/bash
+
+killname /py/qwenvl/app/qwenvl.py
+~/ve/qwenvl/bin/python ~/py/qwenvl/app/qwenvl.py -w ~/py/qwenvl >~/py/qwenvl/logs/stderr.log 2>&1 &
+exit 0
diff --git a/wwwroot/api/index.dspy b/wwwroot/api/index.dspy
new file mode 100644
index 0000000..36881a8
--- /dev/null
+++ b/wwwroot/api/index.dspy
@@ -0,0 +1,3 @@
+info(f'{params_kw=}')
+return inference(params_kw.prompt, image=params_kw.image, video=params_kw.video)
+