367 lines
15 KiB
Python
367 lines
15 KiB
Python
|
import asyncio,logging
|
|||
|
|
from collections import deque
|
|||
|
|
import threading, time,queue,uuid,time,array
|
|||
|
|
from concurrent.futures import ThreadPoolExecutor
|
|||
|
|
|
|||
|
|
ALI_KEY = "sk-a47a3fb5f4a94f66bbaf713779101c75"
|
|||
|
|
from dashscope.api_entities.dashscope_response import SpeechSynthesisResponse
|
|||
|
|
from dashscope.audio.tts import (
|
|||
|
|
ResultCallback as TTSResultCallback,
|
|||
|
|
SpeechSynthesizer as TTSSpeechSynthesizer,
|
|||
|
|
SpeechSynthesisResult as TTSSpeechSynthesisResult,
|
|||
|
|
)
|
|||
|
|
# cyx 2025 01 19 测试cosyvoice 使用tts_v2 版本
|
|||
|
|
from dashscope.audio.tts_v2 import (
|
|||
|
|
ResultCallback as CosyResultCallback,
|
|||
|
|
SpeechSynthesizer as CosySpeechSynthesizer,
|
|||
|
|
AudioFormat,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
class QwenTTS:
|
|||
|
|
def __init__(self, key,format="mp3",sample_rate=44100, model_name="cosyvoice-v1/longyuan"):
|
|||
|
|
import dashscope
|
|||
|
|
import ssl
|
|||
|
|
logging.info(f"---QwenTTS Construtor-- {format} {sample_rate} {model_name}") # cyx
|
|||
|
|
self.model_name = model_name
|
|||
|
|
dashscope.api_key = key
|
|||
|
|
ssl._create_default_https_context = ssl._create_unverified_context # 禁用验证
|
|||
|
|
self.synthesizer = None
|
|||
|
|
self.callback = None
|
|||
|
|
self.is_cosyvoice = False
|
|||
|
|
self.voice = ""
|
|||
|
|
self.format = format
|
|||
|
|
self.sample_rate = sample_rate
|
|||
|
|
if '/' in model_name:
|
|||
|
|
parts = model_name.split('/', 1)
|
|||
|
|
# 返回分离后的两个字符串parts[0], parts[1]
|
|||
|
|
if parts[0] == 'cosyvoice-v1':
|
|||
|
|
self.is_cosyvoice = True
|
|||
|
|
self.voice = parts[1]
|
|||
|
|
|
|||
|
|
class Callback(TTSResultCallback):
|
|||
|
|
def __init__(self) -> None:
|
|||
|
|
self.dque = deque()
|
|||
|
|
|
|||
|
|
def _run(self):
|
|||
|
|
while True:
|
|||
|
|
if not self.dque:
|
|||
|
|
time.sleep(0)
|
|||
|
|
continue
|
|||
|
|
val = self.dque.popleft()
|
|||
|
|
if val:
|
|||
|
|
yield val
|
|||
|
|
else:
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
def on_open(self):
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
def on_complete(self):
|
|||
|
|
self.dque.append(None)
|
|||
|
|
|
|||
|
|
def on_error(self, response: SpeechSynthesisResponse):
|
|||
|
|
print("Qwen tts error", str(response))
|
|||
|
|
raise RuntimeError(str(response))
|
|||
|
|
|
|||
|
|
def on_close(self):
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
def on_event(self, result: TTSSpeechSynthesisResult):
|
|||
|
|
if result.get_audio_frame() is not None:
|
|||
|
|
self.dque.append(result.get_audio_frame())
|
|||
|
|
|
|||
|
|
# --------------------------
|
|||
|
|
|
|||
|
|
class Callback_Cosy(CosyResultCallback):
|
|||
|
|
def __init__(self,on_audio_data) -> None:
|
|||
|
|
self.dque = deque()
|
|||
|
|
self.on_audio_data = on_audio_data
|
|||
|
|
|
|||
|
|
def _run(self):
|
|||
|
|
while True:
|
|||
|
|
if not self.dque:
|
|||
|
|
time.sleep(0)
|
|||
|
|
continue
|
|||
|
|
val = self.dque.popleft()
|
|||
|
|
if val:
|
|||
|
|
yield val
|
|||
|
|
else:
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
def on_open(self):
|
|||
|
|
logging.info("---Qwen tts on_open---")
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
def on_complete(self):
|
|||
|
|
self.dque.append(None)
|
|||
|
|
|
|||
|
|
def on_error(self, response: SpeechSynthesisResponse):
|
|||
|
|
print("Qwen tts error", str(response))
|
|||
|
|
raise RuntimeError(str(response))
|
|||
|
|
|
|||
|
|
def on_close(self):
|
|||
|
|
# print("---Qwen call back close") # cyx
|
|||
|
|
logging.info("---Qwen tts on_close---")
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
""" canceled for test 语音大模型CosyVoice
|
|||
|
|
def on_event(self, result: SpeechSynthesisResult):
|
|||
|
|
if result.get_audio_frame() is not None:
|
|||
|
|
self.dque.append(result.get_audio_frame())
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def on_event(self, message):
|
|||
|
|
# print(f"recv speech synthsis message {message}")
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
# 以下适合语音大模型CosyVoice
|
|||
|
|
def on_data(self, data: bytes) -> None:
|
|||
|
|
if len(data) > 0:
|
|||
|
|
if self.on_audio_data:
|
|||
|
|
self.on_audio_data(data)
|
|||
|
|
else:
|
|||
|
|
self.dque.append(data)
|
|||
|
|
|
|||
|
|
# --------------------------
|
|||
|
|
|
|||
|
|
def tts(self, text):
|
|||
|
|
print(f"--QwenTTS--tts_stream begin-- {text} {self.is_cosyvoice} {self.voice}") # cyx
|
|||
|
|
# text = self.normalize_text(text)
|
|||
|
|
try:
|
|||
|
|
# if self.model_name != 'cosyvoice-v1':
|
|||
|
|
if self.is_cosyvoice is False:
|
|||
|
|
self.callback = self.Callback()
|
|||
|
|
TTSSpeechSynthesizer.call(model=self.model_name,
|
|||
|
|
text=text,
|
|||
|
|
callback=self.callback,
|
|||
|
|
format="wav") # format="mp3")
|
|||
|
|
else:
|
|||
|
|
self.callback = self.Callback_Cosy(None)
|
|||
|
|
format =self.get_audio_format(self.format,self.sample_rate)
|
|||
|
|
self.synthesizer = CosySpeechSynthesizer(
|
|||
|
|
model='cosyvoice-v1',
|
|||
|
|
# voice="longyuan", #"longfei",
|
|||
|
|
voice=self.voice,
|
|||
|
|
callback=self.callback,
|
|||
|
|
format=format
|
|||
|
|
)
|
|||
|
|
self.synthesizer.call(text)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"---dale---20 error {e}") # cyx
|
|||
|
|
# -----------------------------------
|
|||
|
|
try:
|
|||
|
|
for data in self.callback._run():
|
|||
|
|
#logging.info(f"dashcope return data {len(data)}")
|
|||
|
|
yield data
|
|||
|
|
# print(f"---Qwen return data {num_tokens_from_string(text)}")
|
|||
|
|
# yield num_tokens_from_string(text)
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
raise RuntimeError(f"**ERROR**: {e}")
|
|||
|
|
|
|||
|
|
def init_streaming_call(self, on_data):
|
|||
|
|
try:
|
|||
|
|
self.callback = self.Callback_Cosy(on_data)
|
|||
|
|
format =self.get_audio_format(self.format,self.sample_rate)
|
|||
|
|
self.synthesizer = CosySpeechSynthesizer(
|
|||
|
|
model='cosyvoice-v1',
|
|||
|
|
# voice="longyuan", #"longfei",
|
|||
|
|
voice=self.voice,
|
|||
|
|
callback=self.callback,
|
|||
|
|
format=format
|
|||
|
|
)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"---dale---30 error {e}") # cyx
|
|||
|
|
# -----------------------------------
|
|||
|
|
|
|||
|
|
def streaming_call(self,text):
|
|||
|
|
if self.synthesizer:
|
|||
|
|
self.synthesizer.streaming_call(text)
|
|||
|
|
def end_streaming_call(self):
|
|||
|
|
if self.synthesizer:
|
|||
|
|
self.synthesizer.streaming_complete()
|
|||
|
|
|
|||
|
|
def get_audio_format(self, format: str, sample_rate: int):
|
|||
|
|
"""动态获取音频格式"""
|
|||
|
|
from dashscope.audio.tts_v2 import AudioFormat
|
|||
|
|
format_map = {
|
|||
|
|
(8000, 'mp3'): AudioFormat.MP3_8000HZ_MONO_128KBPS,
|
|||
|
|
(8000, 'pcm'): AudioFormat.PCM_8000HZ_MONO_16BIT,
|
|||
|
|
(8000, 'wav'): AudioFormat.WAV_8000HZ_MONO_16BIT,
|
|||
|
|
(16000, 'pcm'): AudioFormat.PCM_16000HZ_MONO_16BIT,
|
|||
|
|
(22050, 'mp3'): AudioFormat.MP3_22050HZ_MONO_256KBPS,
|
|||
|
|
(22050, 'pcm'): AudioFormat.PCM_22050HZ_MONO_16BIT,
|
|||
|
|
(22050, 'wav'): AudioFormat.WAV_22050HZ_MONO_16BIT,
|
|||
|
|
(44100, 'mp3'): AudioFormat.MP3_44100HZ_MONO_256KBPS,
|
|||
|
|
(44100, 'pcm'): AudioFormat.PCM_44100HZ_MONO_16BIT,
|
|||
|
|
(44100, 'wav'): AudioFormat.WAV_44100HZ_MONO_16BIT,
|
|||
|
|
(48000, 'mp3'): AudioFormat.MP3_48000HZ_MONO_256KBPS,
|
|||
|
|
(48000, 'pcm'): AudioFormat.PCM_48000HZ_MONO_16BIT,
|
|||
|
|
(48000, 'wav'):AudioFormat.WAV_48000HZ_MONO_16BIT
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
return format_map.get((sample_rate, format), AudioFormat.MP3_16000HZ_MONO_128KBPS)
|
|||
|
|
|
|||
|
|
class StreamSessionManager:
|
|||
|
|
def __init__(self):
|
|||
|
|
self.sessions = {} # {session_id: {'tts_model': obj, 'buffer': queue, 'task_queue': Queue}}
|
|||
|
|
self.lock = threading.Lock()
|
|||
|
|
self.executor = ThreadPoolExecutor(max_workers=30) # 固定大小线程池
|
|||
|
|
self.gc_interval = 300 # 5分钟清理一次 5 x 60 300秒
|
|||
|
|
self.gc_tts = 10 # 10s 大模型开始输出文本有可能需要比较久，2025年5 24 从3s->10s
|
|||
|
|
|
|||
|
|
def create_session(self, tts_model,sample_rate =8000, stream_format='mp3',voice='cosyvoice-v1/longxiaochun'):
|
|||
|
|
session_id = str(uuid.uuid4())
|
|||
|
|
def on_audio_data(chunk):
|
|||
|
|
session = self.sessions.get(session_id)
|
|||
|
|
first_chunk = not session['tts_chunk_data_valid']
|
|||
|
|
if session['stream_format'] == 'wav':
|
|||
|
|
if first_chunk:
|
|||
|
|
chunk_len = len(chunk)
|
|||
|
|
if chunk_len > 2048:
|
|||
|
|
session['buffer'].put(audio_fade_in(chunk, 1024))
|
|||
|
|
else:
|
|||
|
|
session['buffer'].put(audio_fade_in(chunk, chunk_len))
|
|||
|
|
else:
|
|||
|
|
session['buffer'].put(chunk)
|
|||
|
|
else:
|
|||
|
|
session['buffer'].put(chunk)
|
|||
|
|
session['last_active'] = time.time()
|
|||
|
|
session['audio_chunk_count'] = session['audio_chunk_count'] + 1
|
|||
|
|
if session['tts_chunk_data_valid'] is False:
|
|||
|
|
session['tts_chunk_data_valid'] = True # 20250510 增加，表示连接TTS后台已经返回，可以通知前端了
|
|||
|
|
|
|||
|
|
with self.lock:
|
|||
|
|
ali_tts_model = QwenTTS(ALI_KEY,stream_format, sample_rate,voice.split('@')[0])
|
|||
|
|
self.sessions[session_id] = {
|
|||
|
|
'tts_model': ali_tts_model, #tts_model,
|
|||
|
|
'buffer': queue.Queue(maxsize=300), # 线程安全队列
|
|||
|
|
'task_queue': queue.Queue(),
|
|||
|
|
'active': True,
|
|||
|
|
'last_active': time.time(),
|
|||
|
|
'audio_chunk_count':0,
|
|||
|
|
'finished': threading.Event(), # 添加事件对象
|
|||
|
|
'sample_rate':sample_rate,
|
|||
|
|
'stream_format':stream_format,
|
|||
|
|
"tts_chunk_data_valid":False,
|
|||
|
|
'voice':voice,
|
|||
|
|
}
|
|||
|
|
self.sessions[session_id]['tts_model'].init_streaming_call(on_audio_data)
|
|||
|
|
# 启动任务处理线程
|
|||
|
|
threading.Thread(target=self._process_tasks, args=(session_id,), daemon=True).start()
|
|||
|
|
return session_id
|
|||
|
|
|
|||
|
|
def append_text(self, session_id, text):
|
|||
|
|
with self.lock:
|
|||
|
|
session = self.sessions.get(session_id)
|
|||
|
|
if not session: return
|
|||
|
|
# 将文本放入任务队列（非阻塞）
|
|||
|
|
#logging.info(f"StreamSessionManager append_text {text}")
|
|||
|
|
try:
|
|||
|
|
session['task_queue'].put(text, block=False)
|
|||
|
|
except queue.Full:
|
|||
|
|
logging.warning(f"Session {session_id} task queue full")
|
|||
|
|
|
|||
|
|
def _process_tasks(self, session_id):
|
|||
|
|
"""任务处理线程（每个会话独立）"""
|
|||
|
|
while True:
|
|||
|
|
session = self.sessions.get(session_id)
|
|||
|
|
if not session or not session['active']:
|
|||
|
|
break
|
|||
|
|
try:
|
|||
|
|
#logging.info(f"StreamSessionManager _process_tasks {session['task_queue'].qsize()}")
|
|||
|
|
# 合并多个文本块（最多等待50ms）
|
|||
|
|
texts = []
|
|||
|
|
while len(texts) < 5: # 最大合并5个文本块
|
|||
|
|
try:
|
|||
|
|
text = session['task_queue'].get(timeout=0.1)
|
|||
|
|
#logging.info(f"StreamSessionManager _process_tasks --0 {len(texts)}")
|
|||
|
|
texts.append(text)
|
|||
|
|
except queue.Empty:
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
if texts:
|
|||
|
|
session['last_active'] = time.time() # 如果有处理文本，重置活跃时间
|
|||
|
|
# 提交到线程池处理
|
|||
|
|
#future=self.executor.submit(
|
|||
|
|
# self._generate_audio,
|
|||
|
|
# session_id,
|
|||
|
|
# ' '.join(texts) # 合并文本减少请求次数
|
|||
|
|
#)
|
|||
|
|
#future.result() # 等待转换任务执行完毕
|
|||
|
|
session['tts_model'].streaming_call(''.join(texts))
|
|||
|
|
session['last_active'] = time.time()
|
|||
|
|
# 会话超时检查
|
|||
|
|
if time.time() - session['last_active'] > self.gc_interval:
|
|||
|
|
self.close_session(session_id)
|
|||
|
|
break
|
|||
|
|
if time.time() - session['last_active'] > self.gc_tts:
|
|||
|
|
session['tts_model'].end_streaming_call()
|
|||
|
|
session['finished'].set()
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logging.error(f"Task processing error: {str(e)}")
|
|||
|
|
|
|||
|
|
def _generate_audio(self, session_id, text):
|
|||
|
|
"""实际生成音频（线程池执行）"""
|
|||
|
|
session = self.sessions.get(session_id)
|
|||
|
|
if not session: return
|
|||
|
|
# logging.info(f"_generate_audio:{text}")
|
|||
|
|
first_chunk = True
|
|||
|
|
logging.info(f"转换开始!!! {text}")
|
|||
|
|
try:
|
|||
|
|
for chunk in session['tts_model'].tts(text,session['sample_rate'],session['stream_format']):
|
|||
|
|
if session['stream_format'] == 'wav':
|
|||
|
|
if first_chunk:
|
|||
|
|
chunk_len = len(chunk)
|
|||
|
|
if chunk_len > 2048:
|
|||
|
|
session['buffer'].put(audio_fade_in(chunk,1024))
|
|||
|
|
else:
|
|||
|
|
session['buffer'].put(audio_fade_in(chunk, chunk_len))
|
|||
|
|
first_chunk = False
|
|||
|
|
else:
|
|||
|
|
session['buffer'].put(chunk)
|
|||
|
|
else:
|
|||
|
|
session['buffer'].put(chunk)
|
|||
|
|
session['last_active'] = time.time()
|
|||
|
|
session['audio_chunk_count'] = session['audio_chunk_count'] + 1
|
|||
|
|
if session['tts_chunk_data_valid'] is False:
|
|||
|
|
session['tts_chunk_data_valid'] = True #20250510 增加，表示连接TTS后台已经返回，可以通知前端了
|
|||
|
|
logging.info(f"转换结束!!! {session['audio_chunk_count'] }")
|
|||
|
|
except Exception as e:
|
|||
|
|
session['buffer'].put(f"ERROR:{str(e)}")
|
|||
|
|
logging.info(f"--_generate_audio--error {str(e)}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def close_session(self, session_id):
|
|||
|
|
with self.lock:
|
|||
|
|
if session_id in self.sessions:
|
|||
|
|
logging.info(f"--Session {session_id} close_session")
|
|||
|
|
# 标记会话为不活跃
|
|||
|
|
self.sessions[session_id]['active'] = False
|
|||
|
|
# 延迟2秒后清理资源
|
|||
|
|
threading.Timer(1, self._clean_session, args=[session_id]).start()
|
|||
|
|
|
|||
|
|
def _clean_session(self, session_id):
|
|||
|
|
with self.lock:
|
|||
|
|
if session_id in self.sessions:
|
|||
|
|
del self.sessions[session_id]
|
|||
|
|
|
|||
|
|
def get_session(self, session_id):
|
|||
|
|
return self.sessions.get(session_id)
|
|||
|
|
|
|||
|
|
|
|||
|
|
stream_manager_w_stream = StreamSessionManager()
|
|||
|
|
def audio_fade_in(audio_data, fade_length):
|
|||
|
|
# 假设音频数据是16位单声道PCM
|
|||
|
|
# 将二进制数据转换为整数数组
|
|||
|
|
samples = array.array('h', audio_data)
|
|||
|
|
|
|||
|
|
# 对前fade_length个样本进行淡入处理
|
|||
|
|
for i in range(fade_length):
|
|||
|
|
fade_factor = i / fade_length
|
|||
|
|
samples[i] = int(samples[i] * fade_factor)
|
|||
|
|
|
|||
|
|
# 将整数数组转换回二进制数据
|
|||
|
|
return samples.tobytes()
|