Help with some Text-To-Speech creation

[Mister X]

Heyho,

 

posting this here, as it isn't really a support request for the game itself.

Currently I try to build artificial voices and create voice lines with a combination of OpenVoice v2 and MeloTTS. Based on the provided OpenVoice Jupyter notebook example, I got them up and running locally and altered that code a bit to iterate csv files instead of using a single text line.

 

What I try to achieve now is to plug EmoKnob into this generation process, for better emotion control of the output.

That's the point where I struggle, as their only example is based on another TTS model and with another approach of loading all this stuff.

 

Could somebody please provide me some direction where to adapt my code?

 

Here's my code as I use it right now:

Spoiler

import os
import csv
import torch

from concurrent import futures
import threading

from openvoice import se_extractor
from openvoice.api import ToneColorConverter
from melo.api import TTS
from pydub import AudioSegment, playback

filePath = "D:/Games/Skyrim/Voices/csvFiles/USSEP Changes Reverted and Tweaked.csv"

voices = {
    "riekling": "DLC2RieklingVoice",
    "cicero": "MaleUniqueCicero",
    "adril": "DLC2MaleUniqueAdril",
    "ancano": "MaleUniqueAncano",
    "arngeir": "MaleUniqueArngeir",
    "brynjolf": "MaleUniqueBrynjolf",
    "delvin": "MaleUniqueDelvinMallory",
    "dexion": "DLC1MaleUniqueDexion",
    "esbern": "MaleUniqueEsbern",
    "florentius": "DLC1MaleUniqueFlorentius",
    "galmar": "MaleUniqueGalmar",
    "garan": "DLC1MaleUniqueGaran",
    "gelebor": "DLC1MaleUniqueGelebor",
    "hadvar": "MaleUniqueHadvar",
    "harkon": "DLC1MaleUniqueHarkon",
    "isran": "DLC1MaleUniqueIsran",
    "kodlakwhitemane": "MaleUniqueKodlakWhitemane",
    "lleril": "DLC2MaleUniqueLleril",
    "maleargonian": "MaleArgonian",
    "malebandit": "MaleBandit",
    "malebrute": "MaleBrute",
    "malecommander": "MaleCommander",
    "malecommoner": "MaleCommoner",
    "malecommoneraccented": "MaleCommonerAccented",
    "malecondescending": "MaleCondescending",
    "malecoward": "MaleCoward",
    "maledarkelfcommoner": "DLC2MaleDarkElfCommoner",
    "maledarkelfcynical": "DLC2MaleDarkElfCynical",
    "maledrunk": "MaleDrunk",
    "maledunmer": "MaleDarkElf",
    "maleelfhaughty": "MaleElfHaughty",
    "maleeventoned": "MaleEvenToned",
    "maleeventonedaccented": "MaleEvenTonedAccented",
    "maleguard": "MaleGuard",
    "malekhajiit": "MaleKhajiit",
    "malenord": "MaleNord",
    "malenordcommander": "MaleNordCommander",
    "maleoldgrumpy": "MaleOldGrumpy",
    "maleoldkindly": "MaleOldKindly",
    "maleorc": "MaleOrc",
    "maleslycynical": "MaleSlyCynical",
    "malesoldier": "MaleSoldier",
    "malevampire": "DLC1MaleVampire",
    "malewarlock": "MaleWarlock",
    "maleyoungeager": "MaleYoungEager",
    "mercerfrey": "MaleUniqueMercerFrey",
    "modyn": "DLC2MaleUniqueModyn",
    "nazir": "MaleUniqueNazir",
    "neloth": "DLC2MaleUniqueNeloth",
    "storn": "DLC2MaleUniqueStorn",
    "tulius": "MaleUniqueTullius",
    "ulfric": "MaleUniqueUlfric",
    "astrid": "FemaleUniqueAstrid",
    "delphine": "FemaleUniqueDelphine",
    "elenwen": "FemaleUniqueElenwen",
    "femaleargonian": "FemaleArgonian",
    "femalecommander": "FemaleCommander",
    "femalecommoner": "FemaleCommoner",
    "femalecondescending": "FemaleCondescending",
    "femalecoward": "FemaleCoward",
    "femaledarkelf": "FemaleDarkElf",
    "femaledarkelfcommoner": "DLC2FemaleDarkElfCommoner",
    "femaleelfhaughty": "FemaleElfHaughty",
    "femaleeventoned": "FemaleEvenToned",
    "femalekhajiit": "FemaleKhajiit",
    "femalenord": "FemaleNord",
    "femaleoldgrumpy": "FemaleOldGrumpy",
    "femaleoldkindly": "FemaleOldKindly",
    "femaleorc": "FemaleOrc",
    "femaleshrill": "FemaleShrill",
    "femalesultry": "FemaleSultry",
    "femalevampire": "DLC1FemaleVampire",
    "femaleyoungeager": "FemaleYoungEager",
    "frea": "DLC2FemaleUniqueFrea",
    "fura": "DLC1FemaleUniqueFura",
    "karliah": "FemaleUniqueKarliah",
    "maven": "FemaleUniqueMaven",
    "mirabelleervine": "FemaleUniqueMirabelleErvine",
    "serana": "DLC1SeranaVoice",
    "valerica": "DLC1FemaleUniqueValerica",
    "vex": "FemaleUniqueVex",
    "alduin": "CrUniqueAlduin",
    "dragon": "CrDragonVoice",
    "odahviing": "CrUniqueOdahviing",
    "paarthurnax": "CrUniquePaarthurnax",
}

ckpt_converter = 'openvoice/checkpoints_v2/converter'
device = "cuda:0" if torch.cuda.is_available() else "cpu"

tone_color_converter = ToneColorConverter(f'{ckpt_converter}/config.json', device=device)
tone_color_converter.load_ckpt(f'{ckpt_converter}/checkpoint.pth')

model = TTS(language='EN_NEWEST', device=device)
source_se = torch.load('openvoice/checkpoints_v2/base_speakers/ses/en-newest.pth', map_location=device)

encode_message = "SkyrimAIVoiceClones"

targets = dict()
filePath = os.path.normpath(filePath)

def CreateRefVoices(key):
    reference_voice = f'openvoice/reference/{voices[key]}.wav'
    target_se, audio_name = se_extractor.get_se(reference_voice, tone_color_converter, vad=False)
    targets[key] = target_se

def CreateVoiceLine(line):
    if line[0] in voices and not os.path.isfile(line[4]):
        if not os.path.exists(os.path.dirname(line[4])):
            os.makedirs(os.path.dirname(line[4]))

        src_path = f'openvoice/outputs_v2/tmp_{threading.get_ident()}.wav'
        if not os.path.isfile(src_path):
            tmp = AudioSegment.empty()
            tmp.export(src_path, format="wav")

        if line[0] in targets:
            target_se = targets[line[0]]
        else:
            reference_voice = f'openvoice/reference/{voices[line[0]]}.wav'
            target_se, audio_name = se_extractor.get_se(reference_voice, tone_color_converter, vad=False)
            targets[line[0]] = target_se

        model.tts_to_file(line[1], 0, src_path, speed=1.0, sdp_ratio=0.15, noise_scale=0.1, noise_scale_w=0.4)

        # Run the tone color converter
        tone_color_converter.convert(
            audio_src_path=src_path, 
            src_se=source_se, 
            tgt_se=target_se, 
            output_path=line[4],
            message=encode_message)
        
        os.remove(src_path)

with open(filePath, 'r', encoding="UTF-8") as csvFile:
    csvList = csv.reader(csvFile)

    next(csvList)   # skip header

    with futures.ThreadPoolExecutor(3) as executor:
        executor.map(CreateRefVoices, voices.keys(), timeout=120.0)

    with futures.ThreadPoolExecutor(10) as executor:
        executor.map(CreateVoiceLine, csvList, timeout=30.0)