113 lines
2.8 KiB
JavaScript
113 lines
2.8 KiB
JavaScript
var bricks = window.bricks || {};
|
|
|
|
bricks.VadText = class extends bricks.VBox {
|
|
constructor(opts){
|
|
opts.height = '100%';
|
|
opts.name = opts.name || 'asr_text';
|
|
super(opts);
|
|
this.button = new bricks.Button({
|
|
label:'start',
|
|
icon:bricks_resource('imgs/speak.png')
|
|
});
|
|
this.audio = AudioPlayer({});
|
|
hbox = new bricks.HBox({height:'auto'});
|
|
hbox.add_widget(this.button);
|
|
hbox.add_widget(this.audio)
|
|
this.add_widget(hbox);
|
|
this.filler = new bricks.Filler({});
|
|
this.add_widget(this.filler);
|
|
this.text_w = new bricks.Text({text:' ', wrap:true});
|
|
this.filler.add_widget(this.text_w);
|
|
this.button.bind('click', this.toggle_status.bind(this));
|
|
this.bind('audio_ready', this.handle_audio.bind(this));
|
|
}
|
|
toggle_status(){
|
|
if (this.vad){
|
|
this.stop();
|
|
} else {
|
|
this.start();
|
|
}
|
|
}
|
|
start(){
|
|
this.button.text_w.set_otext('stop');
|
|
schedule_once(this._start.bind(this), 0.1);
|
|
}
|
|
async _start(){
|
|
if (bricks.vad){
|
|
await bricks.vad.stop();
|
|
}
|
|
this.vad = await vad.MicVAD.new({
|
|
onSpeechEnd:(audio) => {
|
|
console.log(audio, this.vad);
|
|
this.dispatch('audio_ready', audio);
|
|
this.handle_audio(audio);
|
|
}
|
|
});
|
|
this.vad.start();
|
|
bricks.vad = this;
|
|
this.text = '';
|
|
}
|
|
stop(){
|
|
this.button.text_w.set_otext('start');
|
|
schedule_once(this._stop.bind(this), 0.1);
|
|
}
|
|
async _stop(){
|
|
await this.vad.pause();
|
|
this.vad = null;
|
|
bricks.vad = null;
|
|
if(this.text != ''){
|
|
this.dispatch('changed', this.getValue());
|
|
}
|
|
}
|
|
async handle_audio(audio){
|
|
console.log('handle_audil() called', audio);
|
|
var wavBuffer = this.floatArrayToWAV(audio, sampleRate=16000);
|
|
var b64audio = this.arrayBufferToBase64(wavBuffer);
|
|
this.audio.set_url('data:audio/wav;base64,' + b64audio);
|
|
var hj = new bricks.HttpJson();
|
|
var d={
|
|
method:'POST',
|
|
params:{
|
|
model:this.model,
|
|
audio:b64audio
|
|
}
|
|
}
|
|
var rj = await hj.httpcall(this.url, d);
|
|
if (rj.status == 'ok'){
|
|
this.text += rj.content
|
|
this.text_w.set_text(this.text);
|
|
} else {
|
|
var w = new bricks.Error({title:'Error',
|
|
timeout:4,
|
|
message:rj.message
|
|
});
|
|
w.open();
|
|
}
|
|
}
|
|
arrayBufferToBase64(wavBuffer) {
|
|
let binary = '';
|
|
const bytes = new Uint8Array(wavBuffer);
|
|
for (let i = 0; i < bytes.byteLength; i++) {
|
|
binary += String.fromCharCode(bytes[i]);
|
|
}
|
|
return btoa(binary);
|
|
}
|
|
floatArrayToWAV(floatArray, sampleRate = 22000) {
|
|
const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
|
|
const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
|
|
buffer.getChannelData(0).set(floatArray);
|
|
|
|
const audioData = buffer;
|
|
|
|
const wavEncoder = new WavEncoder();
|
|
const wavBlob = wavEncoder.encode(audioData);
|
|
return wavBlob;
|
|
}
|
|
getValue(){
|
|
var d = {}
|
|
d[this.name] = this.text;
|
|
}
|
|
}
|
|
|
|
bricks.Factory.register('VadText', bricks.VadText);
|