diff --git a/bricks/asr.js b/bricks/asr.js new file mode 100644 index 0000000..3cf5c92 --- /dev/null +++ b/bricks/asr.js @@ -0,0 +1,62 @@ +var bricks = window.bricks || {}; +bricks.ASRClient = class extends bricks.VBox { + /* + options: + { + start_icon:record.png, + stop_icon:stop.png + ws_url: + } + event: + start: start recording, no params + stop: stop recording, no params + transtext: recognised text, params={ + "content": + "speaker": + "start": + "end": + } + */ + constructor(opts){ + super(opts); + this.icon = new bricks.Icon({url:this.start_icon || bricks_resource('imgs/start_record.png')}); + this.status = 'stop'; + this.icon.bind('click', this.toggle_button.bind(this)); + this.add_widget(this.icon); + this.socket = new WebSocket(this.ws_url); + this.socket.onmessage = this.response_data.bind(this); + this.audioChunks = []; + } + toggle_button(){ + if (this.status == 'stop'){ + this.icon.set_url(this.start_icon||bricks_resource('imgs/start_record.png')); + this.status = 'start'; + this.start_recording(); + } else { + this.icon.set_url(this.stop_icon||bricks_resource('imgs/stop_record.png')); + this.status = 'stop'; + this.stop_recording(); + } + } + async start_recording() { + this.stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + this.mediaRecorder = new MediaRecorder(this.stream); + this.mediaRecorder.ondataavailable = (event) => { + if (event.data.size > 0) { + this.audioChunks.push(event.data); + // 将音频数据通过 WebSocket 发送到服务器 + this.socket.send(event.data); + } + } + this.mediaRecorder.start(1000); // 每 1 秒发送一次数据 + } + stop_recording(){ + this.mediaRecorder.stop(); + } + response_data(event){ + var d = JSON.parse(event.data); + this.dispatch('transtext', d); + } +} + +bricks.Factory.register('ASRClient', bricks.ASRClient); diff --git a/bricks/llm.js b/bricks/llm.js index f611a17..92497ac 100644 --- a/bricks/llm.js +++ b/bricks/llm.js @@ -41,7 +41,7 @@ bricks.LlmMsgAudio = class extends bricks.UpStreaming { return resp; } } -bricks.ModelOutput = class extends bricks.HBox { +bricks.ModelOutput = class extends bricks.VBox { /* { icon: output_view: @@ -53,20 +53,29 @@ bricks.ModelOutput = class extends bricks.HBox { opts = {}; } opts.width = '100%'; + opts.height = 'auto'; super(opts); - this.logid = null; + var hb = new bricks.HBox({width:'100%', cheight:2}); this.img = new bricks.Icon({ rate:2, tip:this.opts.model, url:this.icon||bricks_resource('imgs/llm.png') }); + hb.add_widget(this.img); + var mname = new bricks.Text({text:this.opts.model}); + hb.add_widget(mname); + this.add_widget(hb); + + this.content = new bricks.HBox({width:'100%'}); + this.add_widget(this.content); + this.logid = null; this.run = new bricks.BaseRunning({target:this}); - this.add_widget(this.img); - this.add_widget(this.run); + this.content.add_widget(this.run); this.filler = new bricks.VBox({}); this.filler.set_css('filler'); - this.add_widget(this.filler); - this.add_widget(new bricks.BlankIcon({rate:2, flexShrink:0})); + this.content.add_widget(new bricks.BlankIcon({rate:2, flexShrink:0})); + this.content.add_widget(this.filler); + this.content.add_widget(new bricks.BlankIcon({rate:2, flexShrink:0})); this.build_estimate_widgets(); } build_estimate_widgets(){ @@ -108,7 +117,7 @@ bricks.ModelOutput = class extends bricks.HBox { async update_data(data){ if (this.run) { this.run.stop_timepass(); - this.remove_widget(this.run); + this.content.remove_widget(this.run); if(this.textvoice){ this.upstreaming = new bricks.UpStreaming({ url:this.tts_url @@ -165,15 +174,16 @@ bricks.LlmModel = class extends bricks.JsWidget { this.messages = []; } render_title(){ - var w = new bricks.HBox({}); + var w = new bricks.HBox({padding:'15px'}); w.bind('click', this.show_setup_panel.bind(this)) var img = new bricks.Icon({ rate:2, + tip:this.opts.model, url:this.opts.icon||bricks_resource('imgs/llm.png') }); - var txt = new bricks.Text({text:this.opts.model}); + // var txt = new bricks.Text({text:this.opts.model}); w.add_widget(img); - w.add_widget(txt); + // w.add_widget(txt); return w; } show_setup_panel(event){