From b4c8c816a8c2eedb90b801026a5cd1f475bef173 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Sun, 2 Mar 2025 16:55:08 +0800 Subject: [PATCH] bugfix --- bricks/asr.js | 25 ++++++++++--- bricks/build.sh | 2 +- bricks/utils.js | 10 +++++ bricks/webspeech.js | 73 +++++++++++++++++++++++++++++++++++++ examples/alipay_recharge.ui | 6 +++ examples/asr.ui | 14 +++++++ examples/recharge.ui | 36 ++++++++++++++++++ examples/wechat_recharge.ui | 6 +++ 8 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 bricks/webspeech.js create mode 100644 examples/alipay_recharge.ui create mode 100644 examples/asr.ui create mode 100644 examples/recharge.ui create mode 100644 examples/wechat_recharge.ui diff --git a/bricks/asr.js b/bricks/asr.js index 3cf5c92..5055818 100644 --- a/bricks/asr.js +++ b/bricks/asr.js @@ -6,6 +6,8 @@ bricks.ASRClient = class extends bricks.VBox { start_icon:record.png, stop_icon:stop.png ws_url: + icon_options + ws_params: } event: start: start recording, no params @@ -19,21 +21,26 @@ bricks.ASRClient = class extends bricks.VBox { */ constructor(opts){ super(opts); - this.icon = new bricks.Icon({url:this.start_icon || bricks_resource('imgs/start_record.png')}); + var icon_options = this.icon_options || {}; + icon_options.url = this.start_icon || bricks_resource('imgs/start_recording.png'); + this.icon = new bricks.Icon(icon_options); this.status = 'stop'; this.icon.bind('click', this.toggle_button.bind(this)); this.add_widget(this.icon); this.socket = new WebSocket(this.ws_url); this.socket.onmessage = this.response_data.bind(this); - this.audioChunks = []; + this.bind('transtext', this.response_log.bind(this)); + } + response_log(event){ + console.log('response data=', event.params); } toggle_button(){ if (this.status == 'stop'){ - this.icon.set_url(this.start_icon||bricks_resource('imgs/start_record.png')); + this.icon.set_url(this.start_icon||bricks_resource('imgs/stop_recording.png')); this.status = 'start'; this.start_recording(); } else { - this.icon.set_url(this.stop_icon||bricks_resource('imgs/stop_record.png')); + this.icon.set_url(this.stop_icon||bricks_resource('imgs/start_recording.png')); this.status = 'stop'; this.stop_recording(); } @@ -43,9 +50,15 @@ bricks.ASRClient = class extends bricks.VBox { this.mediaRecorder = new MediaRecorder(this.stream); this.mediaRecorder.ondataavailable = (event) => { if (event.data.size > 0) { - this.audioChunks.push(event.data); // 将音频数据通过 WebSocket 发送到服务器 - this.socket.send(event.data); + blobToBase64(event.data).then((b64str) => { + var d = objcopy(this.ws_params); + d.type = 'audiobuffer'; + d.data = b64str; + this.socket.send(JSON.stringify(d)); + }).catch((error) => { + console.log('Error', error); + }); } } this.mediaRecorder.start(1000); // 每 1 秒发送一次数据 diff --git a/bricks/build.sh b/bricks/build.sh index c8d6ae3..1f4e4f3 100755 --- a/bricks/build.sh +++ b/bricks/build.sh @@ -9,7 +9,7 @@ SOURCES=" page_data_loader.js factory.js uitypesdef.js utils.js uitype.js \ binstreaming.js streaming_audio.js vadtext.js rtc.js docxviewer.js \ llm_dialog.js llm.js websocket.js datarow.js tabular.js \ line.js pie.js bar.js gobang.js period.js iconbarpage.js \ - keypress.js " + keypress.js asr.js webspeech.js " echo ${SOURCES} cat ${SOURCES} > ../dist/bricks.js # uglifyjs --compress --mangle -- ../dist/bricks.js > ../dist/bricks.min.js diff --git a/bricks/utils.js b/bricks/utils.js index 83a1033..a544245 100644 --- a/bricks/utils.js +++ b/bricks/utils.js @@ -554,6 +554,16 @@ bricks.Queue = class { } } +function blobToBase64(blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = function() { + resolve(reader.result); + }; + reader.onerror = reject; + reader.readAsDataURL(blob); + }); +} /* // 使用队列 const queue = new Queue(); diff --git a/bricks/webspeech.js b/bricks/webspeech.js new file mode 100644 index 0000000..e66a63e --- /dev/null +++ b/bricks/webspeech.js @@ -0,0 +1,73 @@ +var bricks = window.bricks || {}; + +bricks.WebTTS = class extends VBox { + constructor(opts){ + super(opts); + } + speak(text){ + // 检查浏览器是否支持SpeechSynthesis + if ('speechSynthesis' in window) { + var utterance = new SpeechSynthesisUtterance(text); + // 设置语音属性,例如语言 + utterance.lang = bricks.app.lang; + utterance.pitch = 1; + utterance.rate = 1; + utterance.onstart = function(event) { + console.log('语音合成开始'); + }; + // 当语音合成结束时触发 + utterance.onend = function(event) { + console.log('语音合成结束'); + }; + // 当语音合成出错时触发 + utterance.onerror = function(event) { + console.error('语音合成出错:', event.error); + }; + // 将utterance添加到语音合成队列 + window.speechSynthesis.speak(utterance); + } else { + console.error('浏览器不支持SpeechSynthesis'); + } + } +} + +bricks.WebASR = class extends VBox { + constructor(opts){ + super(opts); + this.reognition = None; + } + start_recording(){ + // 检查浏览器是否支持SpeechRecognition + if ('SpeechRecognition' in window) { + // 创建SpeechRecognition实例 + this.recognition = new SpeechRecognition(); + // 处理识别错误 + this.recognition.onerror = function(event) { + console.error('识别错误:', event.error); + }; + // 处理语音识别结束事件 + this.recognition.onend = function() { + console.log('语音识别已结束'); + }; + // 处理识别结果 + this.recognition.onresult = function(event) { + var transcript = event.results[0][0].transcript; + this.dispatch('asr_result', { + content:transcript + }); + console.log('识别结果:', transcript); + }; + this.recognition.lang = bricks.app.lang; + this.recognition.start(); + } else { + console.log('browser has not SpeechRecognition'); + } + } + stop_recording(){ + this.recognition.stop(); + } +} + +bricks.Factory.register('WebTTS', bricks.WebTTS); +bricks.Factory.register('WebASR', bricks.WebASR); + diff --git a/examples/alipay_recharge.ui b/examples/alipay_recharge.ui new file mode 100644 index 0000000..d17765d --- /dev/null +++ b/examples/alipay_recharge.ui @@ -0,0 +1,6 @@ +{ + "widgettype":"VBox", + "options":{ + "height":"100%" + } +} diff --git a/examples/asr.ui b/examples/asr.ui new file mode 100644 index 0000000..eb91a06 --- /dev/null +++ b/examples/asr.ui @@ -0,0 +1,14 @@ +{ + "widgettype":"ASRClient", + "options": + { + "ws_url":"{{entire_url('/wss/ws/test.ws')}}?userid={{get_user()}}", + "icon_options":{ + "rate":3 + }, + "ws_params":{ + "kkk":1, + "brade":"123" + } + } +} diff --git a/examples/recharge.ui b/examples/recharge.ui new file mode 100644 index 0000000..c329219 --- /dev/null +++ b/examples/recharge.ui @@ -0,0 +1,36 @@ +{ + "widgettype":"IconbarPage", + "options":{ + "bar_at":"top", + "bar_opts":{ + "margin":"10px", + "rate":1.6, + "tools":[ + { + "name":"wechat", + "icon":"{{entire_url('/imgs/weixinzhifu.png')}}", + "label":"微信充值", + "tip":"使用微信为账户充值", + "content":{ + "widgettype":"urlwidget", + "options":{ + "url":"{{entire_url('wechat_recharge.ui')}}" + } + } + }, + { + "name":"alipay", + "icon":"{{entire_url('/imgs/zhifubao.png')}}", + "label":"支付宝充值", + "tip":"使用支付宝为账户充值", + "content":{ + "widgettype":"urlwidget", + "options":{ + "url":"{{entire_url('alipay_recharge.ui')}}" + } + } + } + ] + } + } +} diff --git a/examples/wechat_recharge.ui b/examples/wechat_recharge.ui new file mode 100644 index 0000000..d17765d --- /dev/null +++ b/examples/wechat_recharge.ui @@ -0,0 +1,6 @@ +{ + "widgettype":"VBox", + "options":{ + "height":"100%" + } +}