bricks/bricks/vadtext.js
2024-08-09 16:47:25 +08:00

113 lines
2.8 KiB
JavaScript

var bricks = window.bricks || {};
bricks.VadText = class extends bricks.VBox {
constructor(opts){
opts.height = '100%';
opts.name = opts.name || 'asr_text';
super(opts);
this.button = new bricks.Button({
label:'start',
icon:bricks_resource('imgs/speak.png')
});
this.audio = AudioPlayer({});
hbox = new bricks.HBox({height:'auto'});
hbox.add_widget(this.button);
hbox.add_widget(this.audio)
this.add_widget(hbox);
this.filler = new bricks.Filler({});
this.add_widget(this.filler);
this.text_w = new bricks.Text({text:' ', wrap:true});
this.filler.add_widget(this.text_w);
this.button.bind('click', this.toggle_status.bind(this));
this.bind('audio_ready', this.handle_audio.bind(this));
}
toggle_status(){
if (this.vad){
this.stop();
} else {
this.start();
}
}
start(){
this.button.text_w.set_otext('stop');
schedule_once(this._start.bind(this), 0.1);
}
async _start(){
if (bricks.vad){
await bricks.vad.stop();
}
this.vad = await vad.MicVAD.new({
onSpeechEnd:(audio) => {
console.log(audio, this.vad);
this.dispatch('audio_ready', audio);
this.handle_audio(audio);
}
});
this.vad.start();
bricks.vad = this;
this.text = '';
}
stop(){
this.button.text_w.set_otext('start');
schedule_once(this._stop.bind(this), 0.1);
}
async _stop(){
await this.vad.pause();
this.vad = null;
bricks.vad = null;
if(this.text != ''){
this.dispatch('changed', this.getValue());
}
}
async handle_audio(audio){
console.log('handle_audil() called', audio);
var wavBuffer = this.floatArrayToWAV(audio, sampleRate=16000);
var b64audio = this.arrayBufferToBase64(wavBuffer);
this.audio.set_url('data:audio/wav;base64,' + b64audio);
var hj = new bricks.HttpJson();
var d={
method:'POST',
params:{
model:this.model,
audio:b64audio
}
}
var rj = await hj.httpcall(this.url, d);
if (rj.status == 'ok'){
this.text += rj.content
this.text_w.set_text(this.text);
} else {
var w = new bricks.Error({title:'Error',
timeout:4,
message:rj.message
});
w.open();
}
}
arrayBufferToBase64(wavBuffer) {
let binary = '';
const bytes = new Uint8Array(wavBuffer);
for (let i = 0; i < bytes.byteLength; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
}
floatArrayToWAV(floatArray, sampleRate = 22000) {
const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
buffer.getChannelData(0).set(floatArray);
const audioData = buffer;
const wavEncoder = new WavEncoder();
const wavBlob = wavEncoder.encode(audioData);
return wavBlob;
}
getValue(){
var d = {}
d[this.name] = this.text;
}
}
bricks.Factory.register('VadText', bricks.VadText);