yumoqing/rtcllm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bugfix

Browse Source
This commit is contained in:
yumoqing 2024-09-13 10:18:40 +08:00
parent 0371c9c671
commit 7de2590d0a
4 changed files with 34 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
rtcllm/a2a.py
View File

@ -74,7 +74,7 @@ class LLMAudioStreamTrack(AudioStreamTrack):
if audio_file is None: if audio_file is None:
print(f'*****{self.__class__.__name__}._feed(),{audio_file=}') print(f'*****{self.__class__.__name__}._feed(),{audio_file=}')
return return
x = self.oi.run_AT_batch_stream(audio_file) abiters = self.oi.run_AT_batch_stream(audio_file)
self.audio_iters.append(x) self.audio_iters.append(abiters)
return x return x

35
rtcllm/aav.py
View File

@ -7,8 +7,10 @@ from aiortc.mediastreams import MediaStreamError
class MyMediaPlayer(MediaPlayer): class MyMediaPlayer(MediaPlayer):
pass pass
class MyTrackBase: class MyTrackBase(MediaStreamTrack):
def __init__(self, source=None): def __init__(self, source=None):
super().__init__()
self.dumb = None
self.source = source self.source = source
self.set_source_track() self.set_source_track()
print(f'{self.kind=}, {self.__class__.__name__}, {dir(self)}') print(f'{self.kind=}, {self.__class__.__name__}, {dir(self)}')
@ -30,13 +32,31 @@ class MyTrackBase:
async def recv(self): async def recv(self):
print(f'============{self.__class__.__name__}, {self.source.duration=}, {self.source.time=}') print(f'============{self.__class__.__name__}, {self.source.duration=}, {self.source.time=}')
if self.source is None: if self.source is None:
return None return await self.dumb.recv()
if self.track.readyState != 'live': try:
self.source_reload() f = await self.track.recv()
f = await self.track.recv() return f
except MediaStreamError:
return await self.dumb.recv()
except Exception as e:
print_exc()
print(f'{e}')
return await self.dumb.recv()
return f return f
class MyAudioStreamTrack(MyTrackBase):
kind = 'audio'
def __init__(self, source=None):
super().__init__()
self.dumb = AudioStreamTrack()
class MyVideoStreamTrack(MyTrackBase):
kind = 'video'
def __init__(self, source=None):
super().__init__()
self.dumb = AudioStreamTrack()
"""
class MyAudioStreamTrack(AudioStreamTrack): class MyAudioStreamTrack(AudioStreamTrack):
def __init__(self, source=None): def __init__(self, source=None):
super().__init__() super().__init__()
@ -116,9 +136,9 @@ class MyVideoStreamTrack(VideoStreamTrack):
class LoopingVideoTrack(VideoStreamTrack): class LoopingVideoTrack(VideoStreamTrack):
""" '''
A video track that loops a video file. A video track that loops a video file.
""" '''
def __init__(self, filename): def __init__(self, filename):
super().__init__() super().__init__()
self.player = MyMediaPlayer(filename) self.player = MyMediaPlayer(filename)
@ -130,4 +150,5 @@ class LoopingVideoTrack(VideoStreamTrack):
self.player = MyMediaPlayer(self.player._file_path) self.player = MyMediaPlayer(self.player._file_path)
frame = await self.player.video.recv() frame = await self.player.video.recv()
return frame return frame
"""

2
rtcllm/rtc.py
View File

@ -138,7 +138,7 @@ class RTCLLM:
pc = peer.pc pc = peer.pc
if track.kind == 'audio': if track.kind == 'audio':
f = partial(self.vad_voiceend, peer) f = partial(self.vad_voiceend, peer)
vadtrack = AudioTrackVad(track, stage=2, onvoiceend=f) vadtrack = AudioTrackVad(track, stage=3, onvoiceend=f)
peer.vadtrack = vadtrack peer.vadtrack = vadtrack
vadtrack.start_vad() vadtrack.start_vad()

4
rtcllm/vad.py
View File

@ -20,7 +20,7 @@ class AudioTrackVad(MediaStreamTrack):
# self.sample_rate = self.track.getSettings().sampleRate # self.sample_rate = self.track.getSettings().sampleRate
# frameSize = self.track.getSettings().frameSize # frameSize = self.track.getSettings().frameSize
# self.frame_duration_ms = (1000 * frameSize) / self.sample_rate # self.frame_duration_ms = (1000 * frameSize) / self.sample_rate
self.frame_duration_ms = 0.00002 self.frame_duration_ms = 0.02
self.num_padding_frames = 20 self.num_padding_frames = 20
self.ring_buffer = collections.deque(maxlen=self.num_padding_frames) self.ring_buffer = collections.deque(maxlen=self.num_padding_frames)
self.triggered = False self.triggered = False
@ -51,6 +51,8 @@ class AudioTrackVad(MediaStreamTrack):
async def recv(self): async def recv(self):
oldf = await self.track.recv() oldf = await self.track.recv()
self.sample_rate = oldf.sample_rate self.sample_rate = oldf.sample_rate
duration = (oldf.samples * 1000) / oldf.sample_rate
print(f'{duration=}, {oldf.samples=}, (oldf.sample_rate=}')
frames = self.resample(oldf) frames = self.resample(oldf)
for f in frames: for f in frames:
if self.debug: if self.debug: