367 lines
15 KiB
Python
367 lines
15 KiB
Python
import asyncio,logging
|
||
from collections import deque
|
||
import threading, time,queue,uuid,time,array
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
|
||
ALI_KEY = "sk-a47a3fb5f4a94f66bbaf713779101c75"
|
||
from dashscope.api_entities.dashscope_response import SpeechSynthesisResponse
|
||
from dashscope.audio.tts import (
|
||
ResultCallback as TTSResultCallback,
|
||
SpeechSynthesizer as TTSSpeechSynthesizer,
|
||
SpeechSynthesisResult as TTSSpeechSynthesisResult,
|
||
)
|
||
# cyx 2025 01 19 测试cosyvoice 使用tts_v2 版本
|
||
from dashscope.audio.tts_v2 import (
|
||
ResultCallback as CosyResultCallback,
|
||
SpeechSynthesizer as CosySpeechSynthesizer,
|
||
AudioFormat,
|
||
)
|
||
|
||
class QwenTTS:
|
||
def __init__(self, key,format="mp3",sample_rate=44100, model_name="cosyvoice-v1/longyuan"):
|
||
import dashscope
|
||
import ssl
|
||
logging.info(f"---QwenTTS Construtor-- {format} {sample_rate} {model_name}") # cyx
|
||
self.model_name = model_name
|
||
dashscope.api_key = key
|
||
ssl._create_default_https_context = ssl._create_unverified_context # 禁用验证
|
||
self.synthesizer = None
|
||
self.callback = None
|
||
self.is_cosyvoice = False
|
||
self.voice = ""
|
||
self.format = format
|
||
self.sample_rate = sample_rate
|
||
if '/' in model_name:
|
||
parts = model_name.split('/', 1)
|
||
# 返回分离后的两个字符串parts[0], parts[1]
|
||
if parts[0] == 'cosyvoice-v1':
|
||
self.is_cosyvoice = True
|
||
self.voice = parts[1]
|
||
|
||
class Callback(TTSResultCallback):
|
||
def __init__(self) -> None:
|
||
self.dque = deque()
|
||
|
||
def _run(self):
|
||
while True:
|
||
if not self.dque:
|
||
time.sleep(0)
|
||
continue
|
||
val = self.dque.popleft()
|
||
if val:
|
||
yield val
|
||
else:
|
||
break
|
||
|
||
def on_open(self):
|
||
pass
|
||
|
||
def on_complete(self):
|
||
self.dque.append(None)
|
||
|
||
def on_error(self, response: SpeechSynthesisResponse):
|
||
print("Qwen tts error", str(response))
|
||
raise RuntimeError(str(response))
|
||
|
||
def on_close(self):
|
||
pass
|
||
|
||
def on_event(self, result: TTSSpeechSynthesisResult):
|
||
if result.get_audio_frame() is not None:
|
||
self.dque.append(result.get_audio_frame())
|
||
|
||
# --------------------------
|
||
|
||
class Callback_Cosy(CosyResultCallback):
|
||
def __init__(self,on_audio_data) -> None:
|
||
self.dque = deque()
|
||
self.on_audio_data = on_audio_data
|
||
|
||
def _run(self):
|
||
while True:
|
||
if not self.dque:
|
||
time.sleep(0)
|
||
continue
|
||
val = self.dque.popleft()
|
||
if val:
|
||
yield val
|
||
else:
|
||
break
|
||
|
||
def on_open(self):
|
||
logging.info("---Qwen tts on_open---")
|
||
pass
|
||
|
||
def on_complete(self):
|
||
self.dque.append(None)
|
||
|
||
def on_error(self, response: SpeechSynthesisResponse):
|
||
print("Qwen tts error", str(response))
|
||
raise RuntimeError(str(response))
|
||
|
||
def on_close(self):
|
||
# print("---Qwen call back close") # cyx
|
||
logging.info("---Qwen tts on_close---")
|
||
pass
|
||
|
||
""" canceled for test 语音大模型CosyVoice
|
||
def on_event(self, result: SpeechSynthesisResult):
|
||
if result.get_audio_frame() is not None:
|
||
self.dque.append(result.get_audio_frame())
|
||
"""
|
||
|
||
def on_event(self, message):
|
||
# print(f"recv speech synthsis message {message}")
|
||
pass
|
||
|
||
# 以下适合语音大模型CosyVoice
|
||
def on_data(self, data: bytes) -> None:
|
||
if len(data) > 0:
|
||
if self.on_audio_data:
|
||
self.on_audio_data(data)
|
||
else:
|
||
self.dque.append(data)
|
||
|
||
# --------------------------
|
||
|
||
def tts(self, text):
|
||
print(f"--QwenTTS--tts_stream begin-- {text} {self.is_cosyvoice} {self.voice}") # cyx
|
||
# text = self.normalize_text(text)
|
||
try:
|
||
# if self.model_name != 'cosyvoice-v1':
|
||
if self.is_cosyvoice is False:
|
||
self.callback = self.Callback()
|
||
TTSSpeechSynthesizer.call(model=self.model_name,
|
||
text=text,
|
||
callback=self.callback,
|
||
format="wav") # format="mp3")
|
||
else:
|
||
self.callback = self.Callback_Cosy(None)
|
||
format =self.get_audio_format(self.format,self.sample_rate)
|
||
self.synthesizer = CosySpeechSynthesizer(
|
||
model='cosyvoice-v1',
|
||
# voice="longyuan", #"longfei",
|
||
voice=self.voice,
|
||
callback=self.callback,
|
||
format=format
|
||
)
|
||
self.synthesizer.call(text)
|
||
except Exception as e:
|
||
print(f"---dale---20 error {e}") # cyx
|
||
# -----------------------------------
|
||
try:
|
||
for data in self.callback._run():
|
||
#logging.info(f"dashcope return data {len(data)}")
|
||
yield data
|
||
# print(f"---Qwen return data {num_tokens_from_string(text)}")
|
||
# yield num_tokens_from_string(text)
|
||
|
||
except Exception as e:
|
||
raise RuntimeError(f"**ERROR**: {e}")
|
||
|
||
def init_streaming_call(self, on_data):
|
||
try:
|
||
self.callback = self.Callback_Cosy(on_data)
|
||
format =self.get_audio_format(self.format,self.sample_rate)
|
||
self.synthesizer = CosySpeechSynthesizer(
|
||
model='cosyvoice-v1',
|
||
# voice="longyuan", #"longfei",
|
||
voice=self.voice,
|
||
callback=self.callback,
|
||
format=format
|
||
)
|
||
except Exception as e:
|
||
print(f"---dale---30 error {e}") # cyx
|
||
# -----------------------------------
|
||
|
||
def streaming_call(self,text):
|
||
if self.synthesizer:
|
||
self.synthesizer.streaming_call(text)
|
||
def end_streaming_call(self):
|
||
if self.synthesizer:
|
||
self.synthesizer.streaming_complete()
|
||
|
||
def get_audio_format(self, format: str, sample_rate: int):
|
||
"""动态获取音频格式"""
|
||
from dashscope.audio.tts_v2 import AudioFormat
|
||
format_map = {
|
||
(8000, 'mp3'): AudioFormat.MP3_8000HZ_MONO_128KBPS,
|
||
(8000, 'pcm'): AudioFormat.PCM_8000HZ_MONO_16BIT,
|
||
(8000, 'wav'): AudioFormat.WAV_8000HZ_MONO_16BIT,
|
||
(16000, 'pcm'): AudioFormat.PCM_16000HZ_MONO_16BIT,
|
||
(22050, 'mp3'): AudioFormat.MP3_22050HZ_MONO_256KBPS,
|
||
(22050, 'pcm'): AudioFormat.PCM_22050HZ_MONO_16BIT,
|
||
(22050, 'wav'): AudioFormat.WAV_22050HZ_MONO_16BIT,
|
||
(44100, 'mp3'): AudioFormat.MP3_44100HZ_MONO_256KBPS,
|
||
(44100, 'pcm'): AudioFormat.PCM_44100HZ_MONO_16BIT,
|
||
(44100, 'wav'): AudioFormat.WAV_44100HZ_MONO_16BIT,
|
||
(48000, 'mp3'): AudioFormat.MP3_48000HZ_MONO_256KBPS,
|
||
(48000, 'pcm'): AudioFormat.PCM_48000HZ_MONO_16BIT,
|
||
(48000, 'wav'):AudioFormat.WAV_48000HZ_MONO_16BIT
|
||
|
||
}
|
||
return format_map.get((sample_rate, format), AudioFormat.MP3_16000HZ_MONO_128KBPS)
|
||
|
||
class StreamSessionManager:
|
||
def __init__(self):
|
||
self.sessions = {} # {session_id: {'tts_model': obj, 'buffer': queue, 'task_queue': Queue}}
|
||
self.lock = threading.Lock()
|
||
self.executor = ThreadPoolExecutor(max_workers=30) # 固定大小线程池
|
||
self.gc_interval = 300 # 5分钟清理一次 5 x 60 300秒
|
||
self.gc_tts = 10 # 10s 大模型开始输出文本有可能需要比较久，2025年5 24 从3s->10s
|
||
|
||
def create_session(self, tts_model,sample_rate =8000, stream_format='mp3',voice='cosyvoice-v1/longxiaochun'):
|
||
session_id = str(uuid.uuid4())
|
||
def on_audio_data(chunk):
|
||
session = self.sessions.get(session_id)
|
||
first_chunk = not session['tts_chunk_data_valid']
|
||
if session['stream_format'] == 'wav':
|
||
if first_chunk:
|
||
chunk_len = len(chunk)
|
||
if chunk_len > 2048:
|
||
session['buffer'].put(audio_fade_in(chunk, 1024))
|
||
else:
|
||
session['buffer'].put(audio_fade_in(chunk, chunk_len))
|
||
else:
|
||
session['buffer'].put(chunk)
|
||
else:
|
||
session['buffer'].put(chunk)
|
||
session['last_active'] = time.time()
|
||
session['audio_chunk_count'] = session['audio_chunk_count'] + 1
|
||
if session['tts_chunk_data_valid'] is False:
|
||
session['tts_chunk_data_valid'] = True # 20250510 增加，表示连接TTS后台已经返回，可以通知前端了
|
||
|
||
with self.lock:
|
||
ali_tts_model = QwenTTS(ALI_KEY,stream_format, sample_rate,voice.split('@')[0])
|
||
self.sessions[session_id] = {
|
||
'tts_model': ali_tts_model, #tts_model,
|
||
'buffer': queue.Queue(maxsize=300), # 线程安全队列
|
||
'task_queue': queue.Queue(),
|
||
'active': True,
|
||
'last_active': time.time(),
|
||
'audio_chunk_count':0,
|
||
'finished': threading.Event(), # 添加事件对象
|
||
'sample_rate':sample_rate,
|
||
'stream_format':stream_format,
|
||
"tts_chunk_data_valid":False,
|
||
'voice':voice,
|
||
}
|
||
self.sessions[session_id]['tts_model'].init_streaming_call(on_audio_data)
|
||
# 启动任务处理线程
|
||
threading.Thread(target=self._process_tasks, args=(session_id,), daemon=True).start()
|
||
return session_id
|
||
|
||
def append_text(self, session_id, text):
|
||
with self.lock:
|
||
session = self.sessions.get(session_id)
|
||
if not session: return
|
||
# 将文本放入任务队列（非阻塞）
|
||
#logging.info(f"StreamSessionManager append_text {text}")
|
||
try:
|
||
session['task_queue'].put(text, block=False)
|
||
except queue.Full:
|
||
logging.warning(f"Session {session_id} task queue full")
|
||
|
||
def _process_tasks(self, session_id):
|
||
"""任务处理线程（每个会话独立）"""
|
||
while True:
|
||
session = self.sessions.get(session_id)
|
||
if not session or not session['active']:
|
||
break
|
||
try:
|
||
#logging.info(f"StreamSessionManager _process_tasks {session['task_queue'].qsize()}")
|
||
# 合并多个文本块（最多等待50ms）
|
||
texts = []
|
||
while len(texts) < 5: # 最大合并5个文本块
|
||
try:
|
||
text = session['task_queue'].get(timeout=0.1)
|
||
#logging.info(f"StreamSessionManager _process_tasks --0 {len(texts)}")
|
||
texts.append(text)
|
||
except queue.Empty:
|
||
break
|
||
|
||
if texts:
|
||
session['last_active'] = time.time() # 如果有处理文本，重置活跃时间
|
||
# 提交到线程池处理
|
||
#future=self.executor.submit(
|
||
# self._generate_audio,
|
||
# session_id,
|
||
# ' '.join(texts) # 合并文本减少请求次数
|
||
#)
|
||
#future.result() # 等待转换任务执行完毕
|
||
session['tts_model'].streaming_call(''.join(texts))
|
||
session['last_active'] = time.time()
|
||
# 会话超时检查
|
||
if time.time() - session['last_active'] > self.gc_interval:
|
||
self.close_session(session_id)
|
||
break
|
||
if time.time() - session['last_active'] > self.gc_tts:
|
||
session['tts_model'].end_streaming_call()
|
||
session['finished'].set()
|
||
break
|
||
|
||
except Exception as e:
|
||
logging.error(f"Task processing error: {str(e)}")
|
||
|
||
def _generate_audio(self, session_id, text):
|
||
"""实际生成音频（线程池执行）"""
|
||
session = self.sessions.get(session_id)
|
||
if not session: return
|
||
# logging.info(f"_generate_audio:{text}")
|
||
first_chunk = True
|
||
logging.info(f"转换开始!!! {text}")
|
||
try:
|
||
for chunk in session['tts_model'].tts(text,session['sample_rate'],session['stream_format']):
|
||
if session['stream_format'] == 'wav':
|
||
if first_chunk:
|
||
chunk_len = len(chunk)
|
||
if chunk_len > 2048:
|
||
session['buffer'].put(audio_fade_in(chunk,1024))
|
||
else:
|
||
session['buffer'].put(audio_fade_in(chunk, chunk_len))
|
||
first_chunk = False
|
||
else:
|
||
session['buffer'].put(chunk)
|
||
else:
|
||
session['buffer'].put(chunk)
|
||
session['last_active'] = time.time()
|
||
session['audio_chunk_count'] = session['audio_chunk_count'] + 1
|
||
if session['tts_chunk_data_valid'] is False:
|
||
session['tts_chunk_data_valid'] = True #20250510 增加，表示连接TTS后台已经返回，可以通知前端了
|
||
logging.info(f"转换结束!!! {session['audio_chunk_count'] }")
|
||
except Exception as e:
|
||
session['buffer'].put(f"ERROR:{str(e)}")
|
||
logging.info(f"--_generate_audio--error {str(e)}")
|
||
|
||
|
||
def close_session(self, session_id):
|
||
with self.lock:
|
||
if session_id in self.sessions:
|
||
logging.info(f"--Session {session_id} close_session")
|
||
# 标记会话为不活跃
|
||
self.sessions[session_id]['active'] = False
|
||
# 延迟2秒后清理资源
|
||
threading.Timer(1, self._clean_session, args=[session_id]).start()
|
||
|
||
def _clean_session(self, session_id):
|
||
with self.lock:
|
||
if session_id in self.sessions:
|
||
del self.sessions[session_id]
|
||
|
||
def get_session(self, session_id):
|
||
return self.sessions.get(session_id)
|
||
|
||
|
||
stream_manager_w_stream = StreamSessionManager()
|
||
def audio_fade_in(audio_data, fade_length):
|
||
# 假设音频数据是16位单声道PCM
|
||
# 将二进制数据转换为整数数组
|
||
samples = array.array('h', audio_data)
|
||
|
||
# 对前fade_length个样本进行淡入处理
|
||
for i in range(fade_length):
|
||
fade_factor = i / fade_length
|
||
samples[i] = int(samples[i] * fade_factor)
|
||
|
||
# 将整数数组转换回二进制数据
|
||
return samples.tobytes() |