diff --git a/bricks/build.sh b/bricks/build.sh index 0590691..f538819 100755 --- a/bricks/build.sh +++ b/bricks/build.sh @@ -7,7 +7,7 @@ SOURCES=" page_data_loader.js factory.js uitypesdef.js utils.js uitype.js \ paging.js datagrid.js iframe.js cols.js echartsext.js \ floaticonbar.js miniform.js wterm.js dynamicaccordion.js \ binstreaming.js streaming_audio.js vadtext.js rtc.js docxviewer.js \ - llm_dialog.js llm.js websocket.js datarow.js tabular.js \ + llm_dialog.js llm.js websocket.js datarow.js tabular.js continueaudio.js \ line.js pie.js bar.js gobang.js period.js iconbarpage.js \ keypress.js asr.js webspeech.js countdown.js progressbar.js " echo ${SOURCES} diff --git a/bricks/camera.js b/bricks/camera.js index 397c331..0f9718a 100644 --- a/bricks/camera.js +++ b/bricks/camera.js @@ -9,6 +9,8 @@ bricks.Camera = class extends bricks.Popup { opts.fps = opts.fps || 60; opts.auto_dismiss = false; super(opts); + this.recordedChunks = []; + this.mediaRecorder = null; this.stream = null; this.video = document.createElement('video'); var filler = new bricks.Filler({}); @@ -83,6 +85,31 @@ bricks.Camera = class extends bricks.Popup { this.task = schedule_once(this.show_picture.bind(this), this.task_period); this.show_cnt += 1; } + videorecorder_start(){ + if (!this.stream) { + throw new Error('Media stream is not initialized. Call init() first.'); + } + this.recordedChunks = []; + this.mediaRecorder = new MediaRecorder(this.stream); + this.mediaRecorder.ondataavailable = (event) => { + if (event.data.size > 0) { + this.recordedChunks.push(event.data); + } + }; + this.mediaRecorder.start(); + } + videorecorder_stop(){ + return new Promise((resolve) => { + this.mediaRecorder.onstop = () => { + const blob = new Blob(this.recordedChunks, { type: 'video/webm' }); + const url = URL.createObjectURL(blob); + resolve({ blob, url }); + }; + this.mediaRecorder.stop(); + this.mediaRecorder = null; + this.recordedChunks = []; + }); + } take_picture(event){ event.stopPropagation(); if (this.task){ diff --git a/bricks/continueaudio.js b/bricks/continueaudio.js index 8c34981..fcabbb9 100644 --- a/bricks/continueaudio.js +++ b/bricks/continueaudio.js @@ -1,6 +1,6 @@ var bricks = window.bricks || {}; -bricks.ContinueAudioPlayer = class extends class bricks.VBox { +bricks.ContinueAudioPlayer = class extends bricks.VBox { constructor(options) { super(options); this.ws_url = options.ws_url; diff --git a/bricks/websocket.js b/bricks/websocket.js index 6ab32ea..824d753 100644 --- a/bricks/websocket.js +++ b/bricks/websocket.js @@ -1,52 +1,75 @@ var bricks = window.bricks || {} bricks.WebSocket = class extends bricks.VBox { + /* + options = { + ws_url: + with_session: + } + event: + onopen: + onmessage: + onerror: + onclose: + ontext: + onbase64audio + onbase64video + */ constructor(opts){ super(opts); - this.dialog = new bricks.VScrollPanel({}) - this.dialog.set_css('filler') - this.input = new bricks.UiStr({ - name:'prompt', - label:'Say something' - }); - this.add_widget(this.dialog); - this.add_widget(this.input); - this.input.bind('keypress', this.check_keypress.bind(this)); - this.ws = new WebSocket(this.ws_url); + if (opts.with_session){ + var session = bricks.app.get_session(); + this.ws = new WebSocket(this.ws_url, sessopn); + } else { + this.ws = new WebSocket(this.ws_url, sessopn); + } this.ws.onopen = function(){ + this.dispatch('onopen'); console.log("open"); } this.ws.onclose = function(e){ - //当客户端收到服务端发送的关闭连接请求时,触发onclose事件 + this.dispatch('onclose'); console.log("close"); } this.ws.onerror = function(e){ - //如果出现连接、处理、接收、发送数据失败的时候触发onerror事件 + this.dispatch('onerror'); console.log(error); } - this.ws.onmessage = this.show_response.bind(this); - } - check_keypress(e){ - if (e.key == 'Enter'){ - var v = this.input.getValue(); - var s = v.prompt; - console.log('check_keypress()', v, s); - if (s != ''){ - this.show_input(s); - } + this.ws.onmessage = function(e){ + var d = JSON.parse(e.data); + var eventname = 'on' + d.type; + this.dispatch(eventname, d.data); } } - show_input(s){ - var w = new bricks.MdWidget({mdtext:s}); - w.set_css('user_msg'); - this.dialog.add_widget(w); - this.ws.send(s); + send_text(text){ + var d = { + type: "text", + data: text + } + this.send(d); } - show_response(e){ - var d = JSON.parse(e.data); - console.log(e.data, d.type, d.data, d); - var w = new bricks.MdWidget({mdtext:d.data}); - w.set_css('llm_msg'); - this.dialog.add_widget(w); + send_base64_video(b64video){ + var d = { + type: "base64video", + data: b64video + } + this.send(d); + } + send_base64_audio(b64audio){ + var d = { + type: "base64audio", + data: b64audio + } + this.send(d); + } + send(d){ + /* d is a object: + { + type: + data: + } + */ + var s = JSON.stringify(d); + this.ws.send(s); } } bricks.Factory.register('WebSocket', bricks.WebSocket);