From 7000471bbe2d1784756958bf0d6f3a00886b4a56 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Wed, 18 Sep 2024 15:28:01 +0800 Subject: [PATCH] bugfix --- rtcllm/rec.py | 70 ++++++++++++ rtcllm/utils.py | 64 +++++++++++ rtcllm/vad.py | 278 ++++++++++++++++++++++-------------------------- 3 files changed, 261 insertions(+), 151 deletions(-) create mode 100644 rtcllm/rec.py create mode 100644 rtcllm/utils.py diff --git a/rtcllm/rec.py b/rtcllm/rec.py new file mode 100644 index 0000000..1ffc34d --- /dev/null +++ b/rtcllm/rec.py @@ -0,0 +1,70 @@ +import pyaudio +import av +import numpy as np +from utils import frames_write_wave +import sys +import select +from vad import MyVad, bytes2frame, to16000_160_frames + +def check_input(): + if select.select([sys.stdin], [], [], 0.0)[0]: + input_str = sys.stdin.readline().strip() + return input_str + else: + return None + +def mic(seconds=None, frames_ms=20): + # PyAudio setup + # + p = pyaudio.PyAudio() + + # Audio format + FORMAT = pyaudio.paInt16 # 16-bit resolution + CHANNELS = 1 # 1 channel for mono + RATE = 16000 # 44.1kHz sampling rate + CHUNK = int(RATE / 100) * 2 # 1024 samples per frame + # PyAudio input stream + stream = p.open(format=FORMAT, + channels=CHANNELS, + rate=RATE, + input=True, + frames_per_buffer=CHUNK) + + print("Recording and streaming audio...") + if seconds is not None: + cnt = int(seconds * 1000 / frames_ms) + else: + cnt = 0 + i = 0 + while True: + if cnt == 0: + if check_input() == 'q': + break + else: + if i>=cnt: + break + # Read audio frames from the microphone + data = stream.read(CHUNK) + # print(f'{data.__class__.__name__}, {len(data)=}, {CHUNK=}') + frames = to16000_160_frames(bytes2frame(data)) + for frame in frames: + yield frame + i += 1 + + # Close the audio stream and the container + stream.close() + p.terminate() + +if __name__ == '__main__': + def cb(f): + print(f'{f} voice wave file') + + i = 0 + vad = MyVad(callback=cb) + for f in mic(): + if i == 0: + print(f'{f.sample_rate=}, {f.samples=}, {f.layout=} ') + i += 1 + vad.vad_check(f) + + print(f'record save to {f}') diff --git a/rtcllm/utils.py b/rtcllm/utils.py new file mode 100644 index 0000000..6e10648 --- /dev/null +++ b/rtcllm/utils.py @@ -0,0 +1,64 @@ +import pyaudio +import av +from av import AudioFrame +import numpy as np + +from appPublic.folderUtils import temp_file + +# 录音参数 + +def MonoMircoPhone(): + CHUNK = 1024 + FORMAT = pyaudio.paInt16 + CHANNELS = 1 + RATE = 44100 + RECORD_SECONDS = 10 + + # 初始化PyAudio + p = pyaudio.PyAudio() + + # 打开音频流 + stream = p.open(format=FORMAT, + channels=CHANNELS, + rate=RATE, + input=True, + frames_per_buffer=CHUNK) + + print("开始录音...") + frames = [] + + for _ in range(0, int(RATE / CHUNK * RECORD_SECONDS)): + data = stream.read(CHUNK) + ndarr = np.frombuffer(data, dtype=np.int16) + print(ndarr.shape) + c = ndarr.shape[0] + ndarr.reshape(c,1) + print(ndarr.shape) + + frame = AudioFrame.from_ndarray(ndarr, format='s16', layout='mono') + yield frame + print(f'{type(data)}') + frames.append(data) + + print("录音结束.") + + # 停止并关闭音频流 + stream.stop_stream() + stream.close() + p.terminate() + +def frames_write_wave(frames): + path = temp_file(suffix='.wav') + output_container = av.open(path, 'w') + out_stream = output_container.add_stream('pcm_s16le') + for frame in frames: + for packet in out_stream.encode(frame): + output_container.mux(packet) + for packet in out_stream.encode(None): + output_container.mux(packet) + output_container.close() + return path + +if __name__ == '__main__': + frames = [f for f in MonoMircoPhone()] + frame_write_wave(frames) diff --git a/rtcllm/vad.py b/rtcllm/vad.py index f23afec..69ad84b 100644 --- a/rtcllm/vad.py +++ b/rtcllm/vad.py @@ -12,71 +12,76 @@ from scipy.io.wavfile import write import numpy as np import av from av import AudioLayout, AudioResampler, AudioFrame, AudioFormat +from utils import frames_write_wave -class AudioTrackVad(MediaStreamTrack): - def __init__(self, track, stage=3, onvoiceend=None): - super().__init__() - self.track = track - self.onvoiceend = onvoiceend - self.vad = webrtcvad.Vad(stage) - # self.sample_rate = self.track.getSettings().sampleRate - # frameSize = self.track.getSettings().frameSize - # self.frame_duration_ms = (1000 * frameSize) / self.sample_rate - self.frame_duration_ms = 0.02 - self.num_padding_frames = 20 - self.ring_buffer = collections.deque(maxlen=self.num_padding_frames) - self.triggered = False +def to16000_160_frames(frame): + remind_byts = b'' + frames = resample(frame, sample_rate=16000) + ret_frames = [] + for f in frames: + if f.samples == 160: + return frames + for f in frames: + b1 = remind_byts + frame2bytes(f) + while len(b1) >= 320: + b = b1[:320] + b1 = b1[320:] + ret_frames.append(bytes2frame(b)) + remind_byts = b1 + return ret_frames + +def bytes2frame(byts, channels=1, sample_rate=16000): + audio_data = np.frombuffer(byts, np.int16) + audio_data = audio_data.reshape((channels, -1)) + layout = 'mono' + if channels == 2: + layout = 'stereo' + # Create an AV frame from the audio data + frame = av.AudioFrame.from_ndarray(audio_data, format='s16', layout='mono') + frame.sample_rate = sample_rate + return frame + +def frame2bytes(frame): + audio_array = frame.to_ndarray() + dtype = audio_array.dtype + audio_bytes = audio_array.tobytes() + return audio_bytes + +def resample(frame, sample_rate=None): + if sample_rate is None: + sample_rate = frame.rate + r = AudioResampler(format='s16', layout='mono', rate=sample_rate) + frame = r.resample(frame) + return frame + +class MyVad(webrtcvad.Vad): + def __init__(self, callback=None): + super().__init__(3) self.voiced_frames = [] - self.loop = asyncio.get_event_loop() - self.task = None - self.debug = True - self.running = False + self.num_padding_frames = 40 + self.ring_buffer = collections.deque(maxlen=self.num_padding_frames) + self.onvoiceend = callback + self.triggered = False + self.cnt = 0 - def start_vad(self): - self.running = True - self.task = self.loop.call_later(self.frame_duration_ms, self._recv) - - def _recv(self): - asyncio.create_task(self.recv()) + def voice_duration(self): + duration = 0 + for f in self.voiced_frames: + duration = f.samples * 1000 / f.sample_rate + duration + return duration - def stop(self): - self.running = False - - def frame2bytes(self, frame): - audio_array = frame.to_ndarray() - dtype = audio_array.dtype - audio_bytes = audio_array.tobytes() - return audio_bytes - - async def recv(self): - frame = await self.track.recv() - self.sample_rate = frame.sample_rate - duration = (frame.samples * 1000) / frame.sample_rate - # print(f'{self.__class__.__name__}.recv(): {duration=}, {frame.samples=}, {frame.sample_rate=}') - try: - await self.vad_check(frame) - except Exception as e: - print(f'{e=}') - print_exc() - return - if self.task: - self.task.cancel() - if self.running: - self.task = self.loop.call_later(self.frame_duration_ms, self._recv) - return frame - - def resample(self, frame, sample_rate=None): - if sample_rate is None: - sample_rate = frame.rate - r = AudioResampler(format='s16', layout='mono', rate=sample_rate) - frame = r.resample(frame) - return frame - - async def vad_check(self, inframe): - frames = self.resample(inframe) - frame = frames[0] - is_speech = self.vad.is_speech(self.frame2bytes(frame), - self.sample_rate) + def vad_check(self, inframe): + """ + ONLY SUPPORT frame with sample_rate = 16000 samples = 160 + """ + frame = inframe + byts = frame2bytes(frame) + if self.cnt == 0: + f = frame + print(f'{f.sample_rate=}, {f.samples=},{f.layout=}, {len(byts)=}') + if not webrtcvad.valid_rate_and_frame_length(frame.sample_rate, frame.samples): + print('ftcygvhbunjiokmpl,mknjbhvgc') + is_speech = self.is_speech(byts, frame.sample_rate, length=frame.samples) if not self.triggered: self.ring_buffer.append((inframe, is_speech)) num_voiced = len([f for f, speech in self.ring_buffer if speech]) @@ -104,102 +109,73 @@ class AudioTrackVad(MediaStreamTrack): if num_unvoiced > 0.9 * self.ring_buffer.maxlen: self.triggered = False duration = self.voice_duration() - if duration > 500 and self.onvoiceend: - ret = await self.write_wave() - await self.onvoiceend(ret) + if duration > 500: + ret = frames_write_wave(self.voiced_frames) + if self.onvoiceend: + self.onvoiceend(ret) else: - print(f'vad sound {duration=}') + print('-----short voice------') + self.ring_buffer.clear() self.voiced_frames = [] + self.cnt += 1 - def to_mono16000_data(self): - lst = [] - for f in self.voiced_frames: - fs = self.resample(f, sample_rate=16000) - lst += fs - audio_data = b''.join([self.frame2bytes(f) for f in lst]) - return audio_data - - async def gen_base64(self): - audio_data = self.to_mono16000_data() - b64 = base64.b64encode(audio_data).decode('utf-8') - return b64 +class AudioTrackVad(MediaStreamTrack): + def __init__(self, track, stage=3, onvoiceend=None): + super().__init__() + self.track = track + self.vad = MyVad(callback=onvoiceend) + # self.sample_rate = self.track.getSettings().sampleRate + # frameSize = self.track.getSettings().frameSize + # self.frame_duration_ms = (1000 * frameSize) / self.sample_rate + self.frame_duration_ms = 0.02 + self.remind_byts = b'' + self.loop = asyncio.get_event_loop() + self.task = None + self.debug = True + self.running = False + + def start_vad(self): + self.running = True + self.task = self.loop.call_later(self.frame_duration_ms, self._recv) - def voice_duration(self): - duration = 0 - for f in self.voiced_frames: - duration = f.samples * 1000 / f.sample_rate + duration - return duration + def _recv(self): + asyncio.create_task(self.recv()) - def frames_resample(self, frames, sr=None): - fs = [] + def stop(self): + self.running = False + + def to16000_160_frames(self, frame): + frames = resample(frame, sample_rate=16000) + ret_frames = [] for f in frames: - fs1 = self.resample(f, sample_rate=sr) - fs += fs1 - return fs + if f.samples == 160: + return frames + for f in frames: + b1 = self.remind_byts + frame2bytes(f) + while len(b1) >= 320: + b = b1[:320] + b1 = b1[320:] + ret_frames.append(bytes2frame(b)) + self.remind_byts = b1 - def frames_write_wave(self, frames): - """ - fb = [ self.frame2bytes(f) for f in frames ] - ndarr = np.frombuffer(b''.join(fb), dtype=np.int16) - fn = temp_file(suffix='.wav') - write(fn, frames[0].sample_rate, ndarr) - return fn - """ - path = temp_file(suffix='.wav') - output_container = av.open(path, 'w') - out_stream = output_container.add_stream('pcm_s16le') - for frame in frames: - for packet in out_stream.encode(frame): - output_container.mux(packet) - for packet in out_stream.encode(None): - output_container.mux(packet) - output_container.close() - return path - - - async def write_wave(self): - """Writes a .wav file. - - Takes path, PCM audio data, and sample rate. - """ - - """ - ############ - # Method:1 - ############ - audio_data = self.to_mono16000_data() - path = temp_file(suffix='.wav') - # print(f'temp_file={path}') - - with contextlib.closing(wave.open(path, 'wb')) as wf: - wf.setnchannels(1) - wf.setsampwidth(2) - wf.setframerate(16000) - wf.writeframes(audio_data) - - # print('************wrote*******') - if self.onvoiceend: - await self.onvoiceend(path) - # print('************over*******') - return - ############ - # Method:2 - ############ - path = temp_file(suffix='.wav') - output_container = av.open(path, 'w') - out_stream = output_container.add_stream('pcm_s16le', rate=16000, layout='mono') - resampler = AudioResampler(format=out_stream.format, layout=out_stream.layout, rate=out_stream.rate) - for frame in self.voiced_frames: - for f in resampler.resample(frame): - output_container.mux(out_stream.encode(f)) - output_container.mux(out_stream.encode()) - output_container.close() - return path - """ - f1 = self.frames_write_wave(self.voiced_frames) - frames = self.frames_resample(self.voiced_frames, sr=16000) - fn = self.frames_write_wave(frames) - print(f'source wave filename={f1}, mono 16000 wave filename={fn}') - return fn + + async def recv(self): + frame = await self.track.recv() + self.sample_rate = frame.sample_rate + duration = (frame.samples * 1000) / frame.sample_rate + # print(f'{self.__class__.__name__}.recv(): {duration=}, {frame.samples=}, {frame.sample_rate=}') + try: + frames = self.to16000_160_frames(frame) + for frame in frames: + await self.vad.vad_check(frame) + except Exception as e: + print(f'{e=}') + print_exc() + return + if self.task: + self.task.cancel() + if self.running: + self.task = self.loop.call_later(self.frame_duration_ms, self._recv) + return frame