bugfix
This commit is contained in:
yumoqing
parent
e80c8a3321
commit
ffaa06c8fc
@ -1,6 +1,7 @@
|
|||||||
{
|
{
|
||||||
"zmq_url" : "tcp://127.0.0.1:10003",
|
"zmq_url" : "tcp://127.0.0.1:10003",
|
||||||
"sample_rate":16000,
|
"sample_rate":16000,
|
||||||
|
"vocab_file":"",
|
||||||
"remove_silence":false,
|
"remove_silence":false,
|
||||||
"modelname":"F5-TTS",
|
"modelname":"F5-TTS",
|
||||||
"device":"cuda:0",
|
"device":"cuda:0",
|
||||||
|
|||||||
265
f5tts.py
265
f5tts.py
@ -12,17 +12,19 @@ from cached_path import cached_path
|
|||||||
|
|
||||||
from model import DiT, UNetT
|
from model import DiT, UNetT
|
||||||
from model.utils_infer import (
|
from model.utils_infer import (
|
||||||
load_vocoder,
|
load_vocoder,
|
||||||
load_model,
|
load_model,
|
||||||
preprocess_ref_audio_text,
|
preprocess_ref_audio_text,
|
||||||
infer_process,
|
infer_process,
|
||||||
remove_silence_for_generated_wav,
|
remove_silence_for_generated_wav,
|
||||||
)
|
)
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
|
from time import time
|
||||||
from appPublic.dictObject import DictObject
|
from appPublic.dictObject import DictObject
|
||||||
from appPublic.zmq_reqrep import ZmqReplier
|
from appPublic.zmq_reqrep import ZmqReplier
|
||||||
|
from appPublic.folderUtils import temp_file
|
||||||
from appPublic.jsonConfig import getConfig
|
from appPublic.jsonConfig import getConfig
|
||||||
|
|
||||||
n_mel_channels = 100
|
n_mel_channels = 100
|
||||||
@ -34,174 +36,43 @@ ode_method = "euler"
|
|||||||
sway_sampling_coef = -1.0
|
sway_sampling_coef = -1.0
|
||||||
speed = 1.0
|
speed = 1.0
|
||||||
|
|
||||||
def main_process(ref_audio, ref_text, text_gen, model_obj, remove_silence):
|
|
||||||
main_voice = {"ref_audio": ref_audio, "ref_text": ref_text}
|
|
||||||
if "voices" not in config:
|
|
||||||
voices = {"main": main_voice}
|
|
||||||
else:
|
|
||||||
voices = config["voices"]
|
|
||||||
voices["main"] = main_voice
|
|
||||||
for voice in voices:
|
|
||||||
voices[voice]["ref_audio"], voices[voice]["ref_text"] = preprocess_ref_audio_text(
|
|
||||||
voices[voice]["ref_audio"], voices[voice]["ref_text"]
|
|
||||||
)
|
|
||||||
print("Voice:", voice)
|
|
||||||
print("Ref_audio:", voices[voice]["ref_audio"])
|
|
||||||
print("Ref_text:", voices[voice]["ref_text"])
|
|
||||||
|
|
||||||
generated_audio_segments = []
|
|
||||||
reg1 = r"(?=\[\w+\])"
|
|
||||||
chunks = re.split(reg1, text_gen)
|
|
||||||
reg2 = r"\[(\w+)\]"
|
|
||||||
for text in chunks:
|
|
||||||
match = re.match(reg2, text)
|
|
||||||
if match:
|
|
||||||
voice = match[1]
|
|
||||||
else:
|
|
||||||
print("No voice tag found, using main.")
|
|
||||||
voice = "main"
|
|
||||||
if voice not in voices:
|
|
||||||
print(f"Voice {voice} not found, using main.")
|
|
||||||
voice = "main"
|
|
||||||
text = re.sub(reg2, "", text)
|
|
||||||
gen_text = text.strip()
|
|
||||||
ref_audio = voices[voice]["ref_audio"]
|
|
||||||
ref_text = voices[voice]["ref_text"]
|
|
||||||
print(f"Voice: {voice}")
|
|
||||||
audio, final_sample_rate, spectragram = infer_process(ref_audio, ref_text, gen_text, model_obj)
|
|
||||||
generated_audio_segments.append(audio)
|
|
||||||
|
|
||||||
if generated_audio_segments:
|
|
||||||
final_wave = np.concatenate(generated_audio_segments)
|
|
||||||
with open(wave_path, "wb") as f:
|
|
||||||
sf.write(f.name, final_wave, final_sample_rate)
|
|
||||||
# Remove silence
|
|
||||||
if remove_silence:
|
|
||||||
remove_silence_for_generated_wav(f.name)
|
|
||||||
print(f.name)
|
|
||||||
|
|
||||||
class F5TTS:
|
class F5TTS:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
config = getConfig()
|
self.config = getConfig()
|
||||||
self.sample_rate = config.sample_rate
|
self.zmq_url = self.config.zmq_url
|
||||||
self.remove_silence = config.remove_silence
|
self.replier = ZmqReplier(self.config.zmq_url, self.generate)
|
||||||
self.modelname = config.modelname
|
# self.vocos = load_vocoder(is_local=True, local_path="../checkpoints/charactr/vocos-mel-24khz")
|
||||||
self.ref_audio_fn = config.ref_audio_fn
|
self.load_model()
|
||||||
self.load_vocoder_from_local = config.is_local or True
|
self.setup_voice()
|
||||||
self.zmq_url = config.zmq_url
|
|
||||||
self.ref_text = config.ref_text
|
|
||||||
self.device = config.device
|
|
||||||
self.cross_fade_duration = config.cross_fade_duration
|
|
||||||
self.gen_ref_audio()
|
|
||||||
self.gen_ref_text()
|
|
||||||
self.replier = ZmqReplier(self.zmq_url, self.generate)
|
|
||||||
self.vocos = load_vocoder(is_local=is_local, local_path="../checkpoints/charactr/vocos-mel-24khz")
|
|
||||||
self.F5TTS_model_cfg = dict(
|
|
||||||
dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4
|
|
||||||
)
|
|
||||||
self.E2TTS_model_cfg = dict(dim=1024, depth=24, heads=16, ff_mult=4)
|
|
||||||
self.model= self.load_model(self.modelname)
|
|
||||||
|
|
||||||
|
|
||||||
def gen_ref_audio(self):
|
|
||||||
"""
|
|
||||||
gen ref_audio
|
|
||||||
"""
|
|
||||||
audio, sr = torchaudio.load(self.ref_audio_fn)
|
|
||||||
if audio.shape[0] > 1:
|
|
||||||
audio = torch.mean(audio, dim=0, keepdim=True)
|
|
||||||
rms = torch.sqrt(torch.mean(torch.square(audio)))
|
|
||||||
if rms < target_rms:
|
|
||||||
audio = audio * target_rms / rms
|
|
||||||
if sr != self.sample_rate:
|
|
||||||
resampler = torchaudio.transforms.Resample(sr, self.sample_rate)
|
|
||||||
audio = resampler(audio)
|
|
||||||
self.ref_audio = audio
|
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
print(f'running {self.zmq_url}')
|
print(f'running {self.zmq_url}')
|
||||||
self.replier._run()
|
self.replier._run()
|
||||||
print('ended ...')
|
print('ended ...')
|
||||||
|
|
||||||
def gen_ref_text(self):
|
def load_model(self):
|
||||||
"""
|
# load models
|
||||||
"""
|
ckpt_file = ''
|
||||||
# Add the functionality to ensure it ends with ". "
|
if self.config.modelname == "F5-TTS":
|
||||||
ref_text = self.ref_text
|
model_cls = DiT
|
||||||
if not ref_text.endswith(". ") and not ref_text.endswith("。"):
|
model_cfg = dict(dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4)
|
||||||
if ref_text.endswith("."):
|
if ckpt_file == "":
|
||||||
ref_text += " "
|
repo_name = "F5-TTS"
|
||||||
else:
|
exp_name = "F5TTS_Base"
|
||||||
ref_text += ". "
|
ckpt_step = 1200000
|
||||||
self.ref_text = ref_text
|
ckpt_file = str(cached_path(f"hf://SWivid/{repo_name}/{exp_name}/model_{ckpt_step}.safetensors"))
|
||||||
|
|
||||||
def load_model(self):
|
elif self.config.modelname == "E2-TTS":
|
||||||
# load models
|
model_cls = UNetT
|
||||||
if self.modelname == "F5-TTS":
|
model_cfg = dict(dim=1024, depth=24, heads=16, ff_mult=4)
|
||||||
model_cls = DiT
|
if ckpt_file == "":
|
||||||
model_cfg = dict(dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4)
|
repo_name = "E2-TTS"
|
||||||
if ckpt_file == "":
|
exp_name = "E2TTS_Base"
|
||||||
repo_name = "F5-TTS"
|
ckpt_step = 1200000
|
||||||
exp_name = "F5TTS_Base"
|
ckpt_file = str(cached_path(f"hf://SWivid/{repo_name}/{exp_name}/model_{ckpt_step}.safetensors"))
|
||||||
ckpt_step = 1200000
|
|
||||||
ckpt_file = str(cached_path(f"hf://SWivid/{repo_name}/{exp_name}/model_{ckpt_step}.safetensors"))
|
|
||||||
|
|
||||||
elif self.modelname == "E2-TTS":
|
self.model = load_model(model_cls, model_cfg, ckpt_file,
|
||||||
model_cls = UNetT
|
self.config.vocab_file).to(self.config.device)
|
||||||
model_cfg = dict(dim=1024, depth=24, heads=16, ff_mult=4)
|
|
||||||
if ckpt_file == "":
|
|
||||||
repo_name = "E2-TTS"
|
|
||||||
exp_name = "E2TTS_Base"
|
|
||||||
ckpt_step = 1200000
|
|
||||||
ckpt_file = str(cached_path(f"hf://SWivid/{repo_name}/{exp_name}/model_{ckpt_step}.safetensors"))
|
|
||||||
|
|
||||||
self.model = load_model(model_cls, model_cfg, ckpt_file, vocab_file)
|
|
||||||
|
|
||||||
def split_text(self, text):
|
|
||||||
max_chars = int(len(self.ref_text.encode('utf-8')) / (self.ref_audio.shape[-1] / self.sample_rate) * (25 - self.ref_audio.shape[-1] / self.sample_rate))
|
|
||||||
gen_text_batches = chunk_text(gen_text, max_chars=max_chars)
|
|
||||||
print('ref_text', ref_text)
|
|
||||||
|
|
||||||
def cross_fade_wave(self, waves):
|
|
||||||
final_wave = generated_waves[0]
|
|
||||||
for i in range(1, len(generated_waves)):
|
|
||||||
prev_wave = final_wave
|
|
||||||
next_wave = generated_waves[i]
|
|
||||||
|
|
||||||
# Calculate cross-fade samples, ensuring it does not exceed wave lengths
|
|
||||||
cross_fade_samples = int(self.cross_fade_duration * self.sample_rate)
|
|
||||||
cross_fade_samples = min(cross_fade_samples, len(prev_wave), len(next_wave))
|
|
||||||
|
|
||||||
if cross_fade_samples <= 0:
|
|
||||||
# No overlap possible, concatenate
|
|
||||||
final_wave = np.concatenate([prev_wave, next_wave])
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Overlapping parts
|
|
||||||
prev_overlap = prev_wave[-cross_fade_samples:]
|
|
||||||
next_overlap = next_wave[:cross_fade_samples]
|
|
||||||
|
|
||||||
# Fade out and fade in
|
|
||||||
fade_out = np.linspace(1, 0, cross_fade_samples)
|
|
||||||
fade_in = np.linspace(0, 1, cross_fade_samples)
|
|
||||||
|
|
||||||
# Cross-faded overlap
|
|
||||||
cross_faded_overlap = prev_overlap * fade_out + next_overlap * fade_in
|
|
||||||
|
|
||||||
# Combine
|
|
||||||
new_wave = np.concatenate([
|
|
||||||
prev_wave[:-cross_fade_samples],
|
|
||||||
cross_faded_overlap,
|
|
||||||
next_wave[cross_fade_samples:]
|
|
||||||
])
|
|
||||||
|
|
||||||
final_wave = new_wave
|
|
||||||
return final_wave
|
|
||||||
|
|
||||||
def write_wave(wave):
|
|
||||||
fn = temp_file(suffix='.wav')
|
|
||||||
sf.write(fn, wave, self.sample_rate)
|
|
||||||
return fn
|
|
||||||
|
|
||||||
def generate(self, d):
|
def generate(self, d):
|
||||||
msg= d.decode('utf-8')
|
msg= d.decode('utf-8')
|
||||||
@ -233,13 +104,75 @@ class F5TTS:
|
|||||||
}
|
}
|
||||||
return json.dumps(d)
|
return json.dumps(d)
|
||||||
|
|
||||||
|
def setup_voice(self):
|
||||||
|
main_voice = {"ref_audio": self.config.ref_audio_fn,
|
||||||
|
"ref_text": self.config.ref_text}
|
||||||
|
if "voices" not in self.config:
|
||||||
|
voices = {"main": main_voice}
|
||||||
|
else:
|
||||||
|
voices = self.config["voices"]
|
||||||
|
voices["main"] = main_voice
|
||||||
|
for voice in voices:
|
||||||
|
voices[voice]["ref_audio"], voices[voice]["ref_text"] = preprocess_ref_audio_text(
|
||||||
|
voices[voice]["ref_audio"], voices[voice]["ref_text"]
|
||||||
|
)
|
||||||
|
print("Voice:", voice)
|
||||||
|
print("Ref_audio:", voices[voice]["ref_audio"])
|
||||||
|
print("Ref_text:", voices[voice]["ref_text"])
|
||||||
|
self.voices = voices
|
||||||
|
|
||||||
|
|
||||||
|
def inference(self, prompt):
|
||||||
|
text_gen = prompt
|
||||||
|
remove_silence = False
|
||||||
|
generated_audio_segments = []
|
||||||
|
reg1 = r"(?=\[\w+\])"
|
||||||
|
chunks = re.split(reg1, text_gen)
|
||||||
|
reg2 = r"\[(\w+)\]"
|
||||||
|
for text in chunks:
|
||||||
|
match = re.match(reg2, text)
|
||||||
|
if match:
|
||||||
|
voice = match[1]
|
||||||
|
else:
|
||||||
|
print("No voice tag found, using main.")
|
||||||
|
voice = "main"
|
||||||
|
if voice not in self.voices:
|
||||||
|
print(f"Voice {voice} not found, using main.")
|
||||||
|
voice = "main"
|
||||||
|
text = re.sub(reg2, "", text)
|
||||||
|
gen_text = text.strip()
|
||||||
|
ref_audio = self.voices[voice]["ref_audio"]
|
||||||
|
ref_text = self.voices[voice]["ref_text"]
|
||||||
|
print(f"Voice: {voice}, {self.model}")
|
||||||
|
audio, final_sample_rate, spectragram = \
|
||||||
|
infer_process(ref_audio, ref_text, gen_text, self.model)
|
||||||
|
generated_audio_segments.append(audio)
|
||||||
|
|
||||||
|
if generated_audio_segments:
|
||||||
|
final_wave = np.concatenate(generated_audio_segments)
|
||||||
|
fn = temp_file(suffix='.wav')
|
||||||
|
with open(fn, "wb") as f:
|
||||||
|
sf.write(f.name, final_wave, final_sample_rate)
|
||||||
|
# Remove silence
|
||||||
|
if remove_silence:
|
||||||
|
remove_silence_for_generated_wav(f.name)
|
||||||
|
return fn
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
workdir = os.getcwd()
|
# workdir = os.getcwd()
|
||||||
config = getConfig(workdir)
|
# config = getConfig(workdir)
|
||||||
tts = F5TTS()
|
tts = F5TTS()
|
||||||
print('here')
|
print('here')
|
||||||
tts.run()
|
# tts.run()
|
||||||
|
while True:
|
||||||
|
print('prompt:')
|
||||||
|
p = input()
|
||||||
|
if p != '':
|
||||||
|
t1 = time()
|
||||||
|
f = tts.inference(p)
|
||||||
|
t2 = time()
|
||||||
|
print(f'{f}, cost {t2-t1} seconds')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user