Upload 73 files

.gitattributes

@@ -30,3 +30,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 image/梁芷柔.png filter=lfs diff=lfs merge=lfs -text
+cleaners/JapaneseCleaner.dll filter=lfs diff=lfs merge=lfs -text
+cleaners/sys.dic filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,124 +1,162 @@
-import re
-import gradio as gr
+import logging
+logging.getLogger('numba').setLevel(logging.WARNING)
+logging.getLogger('matplotlib').setLevel(logging.WARNING)
+logging.getLogger('urllib3').setLevel(logging.WARNING)
+from text import text_to_sequence
+import numpy as np
+from scipy.io import wavfile
 import torch
-import unicodedata
+import json
 import commons
 import utils
-from models import SynthesizerTrn
-from text import text_to_sequence
-
-config_json = "checkpoints/paimeng/config.json"
-pth_path = "checkpoints/paimeng/model.pth"
-
-
-def get_text(text, hps, cleaned=False):
-    if cleaned:
-        text_norm = text_to_sequence(text, hps.symbols, [])
-    else:
-        text_norm = text_to_sequence(text, hps.symbols, hps.data.text_cleaners)
-    if hps.data.add_blank:
+import sys
+import pathlib
+import onnxruntime as ort
+import gradio as gr
+import argparse
+import time
+import os
+from scipy.io.wavfile import write
+
+def is_japanese(string):
+        for ch in string:
+            if ord(ch) > 0x3040 and ord(ch) < 0x30FF:
+                return True
+        return False
+
+def is_english(string):
+        import re
+        pattern = re.compile('^[A-Za-z0-9.,:;!?()_*"\' ]+$')
+        if pattern.fullmatch(string):
+            return True
+        else:
+            return False
+
+def to_numpy(tensor: torch.Tensor):
+    return tensor.detach().cpu().numpy() if tensor.requires_grad \
+        else tensor.detach().numpy()
+
+def get_symbols_from_json(path):
+    assert os.path.isfile(path)
+    with open(path, 'r') as f:
+        data = json.load(f)
+    return data['symbols']
+
+def sle(language,text):
+        text = text.replace('\n','。').replace(' ',',')
+        if language == "中文":
+            tts_input1 = "[ZH]" + text + "[ZH]"
+            return tts_input1
+        elif language == "自动":
+            tts_input1 = f"[JA]{text}[JA]" if is_japanese(text) else f"[ZH]{text}[ZH]"
+            return tts_input1
+        elif language == "日文":
+            tts_input1 = "[JA]" + text + "[JA]"
+            return tts_input1
+        elif language == "英文":
+            tts_input1 = "[EN]" + text + "[EN]"
+            return tts_input1
+        elif language == "手动":
+            return text
+
+def get_text(text,hps_ms):
+    text_norm = text_to_sequence(text,hps_ms.data.text_cleaners)
+    if hps_ms.data.add_blank:
         text_norm = commons.intersperse(text_norm, 0)
     text_norm = torch.LongTensor(text_norm)
     return text_norm
 
-
-def get_label(text, label):
-    if f'[{label}]' in text:
-        return True, text.replace(f'[{label}]', '')
-    else:
-        return False, text
-
-
-def clean_text(text):
-    print(text)
-    jap = re.compile(r'[\u3040-\u309F\u30A0-\u30FF]')  # 匹配日文
-    text = unicodedata.normalize('NFKC', text)
-    text = f"[JA]{text}[JA]" if jap.search(text) else f"[ZH]{text}[ZH]"
-    return text
-
-
-def load_model(config_json, pth_path):
-    dev = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-    hps_ms = utils.get_hparams_from_file(f"{config_json}")
-    n_speakers = hps_ms.data.n_speakers if 'n_speakers' in hps_ms.data.keys() else 0
-    n_symbols = len(hps_ms.symbols) if 'symbols' in hps_ms.keys() else 0
-    net_g_ms = SynthesizerTrn(
-        n_symbols,
-        hps_ms.data.filter_length // 2 + 1,
-        hps_ms.train.segment_size // hps_ms.data.hop_length,
-        n_speakers=n_speakers,
-        **hps_ms.model).to(dev)
-    _ = net_g_ms.eval()
-    _ = utils.load_checkpoint(pth_path, net_g_ms)
-    return net_g_ms
-
-net_g_ms = load_model(config_json, pth_path)
-
-def selection(speaker):
-    if speaker == "南小鸟":
-        spk = 0
-        return spk
-
-    elif speaker == "园田海未":
-        spk = 1
-        return spk
-
-    elif speaker == "小泉花阳":
-        spk = 2
-        return spk
-
-    elif speaker == "星空凛":
-        spk = 3
-        return spk
-
-    elif speaker == "东条希":
-        spk = 4
-        return spk
-
-    elif speaker == "矢泽妮可":
-        spk = 5
-        return spk
-
-    elif speaker == "绚濑绘里":
-        spk = 6
-        return spk
-
-    elif speaker == "西木野真姬":
-        spk = 7
-        return spk
-
-    elif speaker == "高坂穗乃果":
-        spk = 8
-        return spk
-
-def infer(text,speaker_id, n_scale= 0.667,n_scale_w = 0.8, l_scale = 1 ):
-    text = clean_text(text)
-    speaker_id = int(selection(speaker_id))
-    dev = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-    hps_ms = utils.get_hparams_from_file(f"{config_json}")
-    with torch.no_grad():
-        stn_tst = get_text(text, hps_ms, cleaned=False)
-        x_tst = stn_tst.unsqueeze(0).to(dev)
-        x_tst_lengths = torch.LongTensor([stn_tst.size(0)]).to(dev)
-        sid = torch.LongTensor([speaker_id]).to(dev)
-        audio = net_g_ms.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=n_scale, noise_scale_w=n_scale_w, length_scale=l_scale)[0][
-            0, 0].data.cpu().float().numpy()
-    return (hps_ms.data.sampling_rate, audio)
-
-idols = ["南小鸟","园田海未","小泉花阳","星空凛","东条希","矢泽妮可","绚濑绘里","西木野真姬","高坂穗乃果"]
-app = gr.Blocks()
-with app:
-    with gr.Tabs():
-
-        with gr.TabItem("Basic"):
-
-            tts_input1 = gr.TextArea(label="请输入纯中文或纯日文", value="大家好")
-            para_input1 = gr.Slider(minimum= 0.01,maximum=1.0,label="更改噪声比例", value=0.667)
-            para_input2 = gr.Slider(minimum= 0.01,maximum=1.0,label="更改噪声偏差", value=0.8)
-            para_input3 = gr.Slider(minimum= 0.1,maximum=10,label="更改时间比例", value=1)
-            tts_submit = gr.Button("Generate", variant="primary")
-            speaker1 = gr.Dropdown(label="选择说话人",choices=idols, value="高坂穗乃果", interactive=True)
-            tts_output2 = gr.Audio(label="Output")
-
-            tts_submit.click(infer, [tts_input1,speaker1,para_input1,para_input2,para_input3], [tts_output2])
+def create_tts_fn(ort_sess, speaker_id):
+    def tts_fn(text , language, n_scale= 0.667,n_scale_w = 0.8, l_scale = 1 ):
+        text =sle(language,text)
+        seq = text_to_sequence(text, cleaner_names=hps.data.text_cleaners)
+        if hps.data.add_blank:
+            seq = commons.intersperse(seq, 0)
+        with torch.no_grad():
+            x = np.array([seq], dtype=np.int64)
+            x_len = np.array([x.shape[1]], dtype=np.int64)
+            sid = np.array([speaker_id], dtype=np.int64)
+            scales = np.array([n_scale, n_scale_w, l_scale], dtype=np.float32)
+            scales.resize(1, 3)
+            ort_inputs = {
+                        'input': x,
+                        'input_lengths': x_len,
+                        'scales': scales,
+                        'sid': sid
+                    }
+            t1 = time.time()
+            audio = np.squeeze(ort_sess.run(None, ort_inputs))
+            audio *= 32767.0 / max(0.01, np.max(np.abs(audio))) * 0.6
+            audio = np.clip(audio, -32767.0, 32767.0)
+            t2 = time.time()
+            spending_time = "推理时间："+str(t2-t1)+"s" 
+            print(spending_time)
+        return (hps.data.sampling_rate, audio) 
+    return tts_fn
+
+
+if __name__ == '__main__':
+    symbols = get_symbols_from_json('checkpoints/Nijigasaki/config.json')
+    hps = utils.get_hparams_from_file('checkpoints/Nijigasaki/config.json')
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    models = []
+    schools = ["ShojoKageki-Nijigasaki","ShojoKageki","Nijigasaki"]
+    lan = ["中文","日文","自动","手动"]
+    with open("checkpoints/info.json", "r", encoding="utf-8") as f:
+        models_info = json.load(f)
+    for i in models_info:
+        school = models_info[i]
+        speakers = school["speakers"]
+        checkpoint = school["checkpoint"]
+        phone_dict = {
+            symbol: i for i, symbol in enumerate(symbols)
+        }
+        ort_sess = ort.InferenceSession(checkpoint)
+        content = []
+        for j in speakers:
+            sid = int(speakers[j]['sid'])
+            title = school
+            example = speakers[j]['speech']
+            name = speakers[j]["name"]
+            content.append((sid, name, title, example, create_tts_fn(ort_sess, sid)))
+        models.append(content)
+    
+    with gr.Blocks() as app:
+        gr.Markdown(
+            "# <center> vits-models\n"
+        )
+        with gr.Tabs():
+            for i in schools:
+                with gr.TabItem(i):
+                    for (sid, name,  title, example, tts_fn) in models[schools.index(i)]:
+                        with gr.TabItem(name):
+                            '''
+                            with gr.Row():
+                                gr.Markdown(
+                                    '<div align="center">'
+                                    f'<a><strong>{name}</strong></a>'
+                                    f'<img style="width:auto;height:300px;" src="file/{sid}.png">' 
+                                    '</div>'
+                                )
+                            '''
+                            with gr.Row():
+                                with gr.Column():
+                                    with gr.Row():
+                                        with gr.Column():
+                                            gr.Markdown(
+                                                '<div align="center">'
+                                                f'<a><strong>{name}</strong></a>'
+                                                f'<img style="width:auto;height:400px;" src="file/image/{name}.png">' 
+                                                '</div>'
+                                            )
+                                            input2 = gr.Dropdown(label="Language", choices=lan, value="自动", interactive=True)
+                                        with gr.Column():
+                                            input1 = gr.TextArea(label="Text", value=example)
+                                            input4 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声比例(noise scale)，以控制情感", value=0.6)
+                                            input5 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声偏差(noise scale w)，以控制音素长短", value=0.668)
+                                            input6 = gr.Slider(minimum=0.1, maximum=10, label="duration", value=1)
+                                            btnVC = gr.Button("Submit")
+                                            output1 = gr.Audio(label="采样率22050")
+                                            
+                            btnVC.click(tts_fn, inputs=[input1, input2, input4, input5, input6], outputs=[output1])
     app.launch()

attentions.py

@@ -1,300 +1,392 @@
 import math
+
 import torch
 from torch import nn
 from torch.nn import functional as F
 
 import commons
 from modules import LayerNorm
-   
+
 
 class Encoder(nn.Module):
-  def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, **kwargs):
-    super().__init__()
-    self.hidden_channels = hidden_channels
-    self.filter_channels = filter_channels
-    self.n_heads = n_heads
-    self.n_layers = n_layers
-    self.kernel_size = kernel_size
-    self.p_dropout = p_dropout
-    self.window_size = window_size
-
-    self.drop = nn.Dropout(p_dropout)
-    self.attn_layers = nn.ModuleList()
-    self.norm_layers_1 = nn.ModuleList()
-    self.ffn_layers = nn.ModuleList()
-    self.norm_layers_2 = nn.ModuleList()
-    for i in range(self.n_layers):
-      self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size))
-      self.norm_layers_1.append(LayerNorm(hidden_channels))
-      self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout))
-      self.norm_layers_2.append(LayerNorm(hidden_channels))
-
-  def forward(self, x, x_mask):
-    attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
-    x = x * x_mask
-    for i in range(self.n_layers):
-      y = self.attn_layers[i](x, x, attn_mask)
-      y = self.drop(y)
-      x = self.norm_layers_1[i](x + y)
-
-      y = self.ffn_layers[i](x, x_mask)
-      y = self.drop(y)
-      x = self.norm_layers_2[i](x + y)
-    x = x * x_mask
-    return x
+    def __init__(self,
+                 hidden_channels,
+                 filter_channels,
+                 n_heads,
+                 n_layers,
+                 kernel_size=1,
+                 p_dropout=0.,
+                 window_size=4,
+                 **kwargs):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+
+        self.drop = nn.Dropout(p_dropout)
+        self.attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.attn_layers.append(
+                MultiHeadAttention(hidden_channels,
+                                   hidden_channels,
+                                   n_heads,
+                                   p_dropout=p_dropout,
+                                   window_size=window_size))
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout))
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask):
+        attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.attn_layers[i](x, x, attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
 
 
 class Decoder(nn.Module):
-  def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs):
-    super().__init__()
-    self.hidden_channels = hidden_channels
-    self.filter_channels = filter_channels
-    self.n_heads = n_heads
-    self.n_layers = n_layers
-    self.kernel_size = kernel_size
-    self.p_dropout = p_dropout
-    self.proximal_bias = proximal_bias
-    self.proximal_init = proximal_init
-
-    self.drop = nn.Dropout(p_dropout)
-    self.self_attn_layers = nn.ModuleList()
-    self.norm_layers_0 = nn.ModuleList()
-    self.encdec_attn_layers = nn.ModuleList()
-    self.norm_layers_1 = nn.ModuleList()
-    self.ffn_layers = nn.ModuleList()
-    self.norm_layers_2 = nn.ModuleList()
-    for i in range(self.n_layers):
-      self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init))
-      self.norm_layers_0.append(LayerNorm(hidden_channels))
-      self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
-      self.norm_layers_1.append(LayerNorm(hidden_channels))
-      self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
-      self.norm_layers_2.append(LayerNorm(hidden_channels))
-
-  def forward(self, x, x_mask, h, h_mask):
-    """
+    def __init__(self,
+                 hidden_channels,
+                 filter_channels,
+                 n_heads,
+                 n_layers,
+                 kernel_size=1,
+                 p_dropout=0.,
+                 proximal_bias=False,
+                 proximal_init=True,
+                 **kwargs):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+
+        self.drop = nn.Dropout(p_dropout)
+        self.self_attn_layers = nn.ModuleList()
+        self.norm_layers_0 = nn.ModuleList()
+        self.encdec_attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.self_attn_layers.append(
+                MultiHeadAttention(hidden_channels,
+                                   hidden_channels,
+                                   n_heads,
+                                   p_dropout=p_dropout,
+                                   proximal_bias=proximal_bias,
+                                   proximal_init=proximal_init))
+            self.norm_layers_0.append(LayerNorm(hidden_channels))
+            self.encdec_attn_layers.append(
+                MultiHeadAttention(hidden_channels,
+                                   hidden_channels,
+                                   n_heads,
+                                   p_dropout=p_dropout))
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout,
+                    causal=True))
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask, h, h_mask):
+        """
     x: decoder input
     h: encoder output
     """
-    self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
-    encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
-    x = x * x_mask
-    for i in range(self.n_layers):
-      y = self.self_attn_layers[i](x, x, self_attn_mask)
-      y = self.drop(y)
-      x = self.norm_layers_0[i](x + y)
-
-      y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
-      y = self.drop(y)
-      x = self.norm_layers_1[i](x + y)
-      
-      y = self.ffn_layers[i](x, x_mask)
-      y = self.drop(y)
-      x = self.norm_layers_2[i](x + y)
-    x = x * x_mask
-    return x
+        self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(
+            device=x.device, dtype=x.dtype)
+        encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.self_attn_layers[i](x, x, self_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_0[i](x + y)
+
+            y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
 
 
 class MultiHeadAttention(nn.Module):
-  def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False):
-    super().__init__()
-    assert channels % n_heads == 0
-
-    self.channels = channels
-    self.out_channels = out_channels
-    self.n_heads = n_heads
-    self.p_dropout = p_dropout
-    self.window_size = window_size
-    self.heads_share = heads_share
-    self.block_length = block_length
-    self.proximal_bias = proximal_bias
-    self.proximal_init = proximal_init
-    self.attn = None
-
-    self.k_channels = channels // n_heads
-    self.conv_q = nn.Conv1d(channels, channels, 1)
-    self.conv_k = nn.Conv1d(channels, channels, 1)
-    self.conv_v = nn.Conv1d(channels, channels, 1)
-    self.conv_o = nn.Conv1d(channels, out_channels, 1)
-    self.drop = nn.Dropout(p_dropout)
-
-    if window_size is not None:
-      n_heads_rel = 1 if heads_share else n_heads
-      rel_stddev = self.k_channels**-0.5
-      self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
-      self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
-
-    nn.init.xavier_uniform_(self.conv_q.weight)
-    nn.init.xavier_uniform_(self.conv_k.weight)
-    nn.init.xavier_uniform_(self.conv_v.weight)
-    if proximal_init:
-      with torch.no_grad():
-        self.conv_k.weight.copy_(self.conv_q.weight)
-        self.conv_k.bias.copy_(self.conv_q.bias)
-      
-  def forward(self, x, c, attn_mask=None):
-    q = self.conv_q(x)
-    k = self.conv_k(c)
-    v = self.conv_v(c)
-    
-    x, self.attn = self.attention(q, k, v, mask=attn_mask)
-
-    x = self.conv_o(x)
-    return x
-
-  def attention(self, query, key, value, mask=None):
-    # reshape [b, d, t] -> [b, n_h, t, d_k]
-    b, d, t_s, t_t = (*key.size(), query.size(2))
-    query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
-    key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
-    value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
-
-    scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
-    if self.window_size is not None:
-      assert t_s == t_t, "Relative attention is only available for self-attention."
-      key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
-      rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings)
-      scores_local = self._relative_position_to_absolute_position(rel_logits)
-      scores = scores + scores_local
-    if self.proximal_bias:
-      assert t_s == t_t, "Proximal bias is only available for self-attention."
-      scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
-    if mask is not None:
-      scores = scores.masked_fill(mask == 0, -1e4)
-      if self.block_length is not None:
-        assert t_s == t_t, "Local attention is only available for self-attention."
-        block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
-        scores = scores.masked_fill(block_mask == 0, -1e4)
-    p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
-    p_attn = self.drop(p_attn)
-    output = torch.matmul(p_attn, value)
-    if self.window_size is not None:
-      relative_weights = self._absolute_position_to_relative_position(p_attn)
-      value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
-      output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
-    output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t]
-    return output, p_attn
-
-  def _matmul_with_relative_values(self, x, y):
-    """
+    def __init__(self,
+                 channels,
+                 out_channels,
+                 n_heads,
+                 p_dropout=0.,
+                 window_size=None,
+                 heads_share=True,
+                 block_length=None,
+                 proximal_bias=False,
+                 proximal_init=False):
+        super().__init__()
+        assert channels % n_heads == 0
+
+        self.channels = channels
+        self.out_channels = out_channels
+        self.n_heads = n_heads
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+        self.heads_share = heads_share
+        self.block_length = block_length
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+        self.attn = None
+
+        self.k_channels = channels // n_heads
+        self.conv_q = nn.Conv1d(channels, channels, 1)
+        self.conv_k = nn.Conv1d(channels, channels, 1)
+        self.conv_v = nn.Conv1d(channels, channels, 1)
+        self.conv_o = nn.Conv1d(channels, out_channels, 1)
+        self.drop = nn.Dropout(p_dropout)
+
+        if window_size is not None:
+            n_heads_rel = 1 if heads_share else n_heads
+            rel_stddev = self.k_channels**-0.5
+            self.emb_rel_k = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev)
+            self.emb_rel_v = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev)
+
+        nn.init.xavier_uniform_(self.conv_q.weight)
+        nn.init.xavier_uniform_(self.conv_k.weight)
+        nn.init.xavier_uniform_(self.conv_v.weight)
+        if proximal_init:
+            with torch.no_grad():
+                self.conv_k.weight.copy_(self.conv_q.weight)
+                self.conv_k.bias.copy_(self.conv_q.bias)
+
+    def forward(self, x, c, attn_mask=None):
+        q = self.conv_q(x)
+        k = self.conv_k(c)
+        v = self.conv_v(c)
+
+        x, self.attn = self.attention(q, k, v, mask=attn_mask)
+
+        x = self.conv_o(x)
+        return x
+
+    def attention(self, query, key, value, mask=None):
+        # reshape [b, d, t] -> [b, n_h, t, d_k]
+        b, d, t_s, t_t = (*key.size(), query.size(2))
+        query = query.view(b, self.n_heads, self.k_channels,
+                           t_t).transpose(2, 3)
+        key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+        value = value.view(b, self.n_heads, self.k_channels,
+                           t_s).transpose(2, 3)
+
+        scores = torch.matmul(query / math.sqrt(self.k_channels),
+                              key.transpose(-2, -1))
+        if self.window_size is not None:
+            msg = "Relative attention is only available for self-attention."
+            assert t_s == t_t, msg
+            key_relative_embeddings = self._get_relative_embeddings(
+                self.emb_rel_k, t_s)
+            rel_logits = self._matmul_with_relative_keys(
+                query / math.sqrt(self.k_channels), key_relative_embeddings)
+            scores_local = self._relative_position_to_absolute_position(
+                rel_logits)
+            scores = scores + scores_local
+        if self.proximal_bias:
+            msg = "Proximal bias is only available for self-attention."
+            assert t_s == t_t, msg
+            scores = scores + self._attention_bias_proximal(t_s).to(
+                device=scores.device, dtype=scores.dtype)
+        if mask is not None:
+            scores = scores.masked_fill(mask == 0, -1e4)
+            if self.block_length is not None:
+                msg = "Local attention is only available for self-attention."
+                assert t_s == t_t, msg
+                block_mask = torch.ones_like(scores).triu(
+                    -self.block_length).tril(self.block_length)
+                scores = scores.masked_fill(block_mask == 0, -1e4)
+        p_attn = F.softmax(scores, dim=-1)  # [b, n_h, t_t, t_s]
+        p_attn = self.drop(p_attn)
+        output = torch.matmul(p_attn, value)
+        if self.window_size is not None:
+            relative_weights = self._absolute_position_to_relative_position(
+                p_attn)
+            value_relative_embeddings = self._get_relative_embeddings(
+                self.emb_rel_v, t_s)
+            output = output + self._matmul_with_relative_values(
+                relative_weights, value_relative_embeddings)
+        output = output.transpose(2, 3).contiguous().view(
+            b, d, t_t)  # [b, n_h, t_t, d_k] -> [b, d, t_t]
+        return output, p_attn
+
+    def _matmul_with_relative_values(self, x, y):
+        """
     x: [b, h, l, m]
     y: [h or 1, m, d]
     ret: [b, h, l, d]
     """
-    ret = torch.matmul(x, y.unsqueeze(0))
-    return ret
+        ret = torch.matmul(x, y.unsqueeze(0))
+        return ret
 
-  def _matmul_with_relative_keys(self, x, y):
-    """
+    def _matmul_with_relative_keys(self, x, y):
+        """
     x: [b, h, l, d]
     y: [h or 1, m, d]
     ret: [b, h, l, m]
     """
-    ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
-    return ret
-
-  def _get_relative_embeddings(self, relative_embeddings, length):
-    max_relative_position = 2 * self.window_size + 1
-    # Pad first before slice to avoid using cond ops.
-    pad_length = max(length - (self.window_size + 1), 0)
-    slice_start_position = max((self.window_size + 1) - length, 0)
-    slice_end_position = slice_start_position + 2 * length - 1
-    if pad_length > 0:
-      padded_relative_embeddings = F.pad(
-          relative_embeddings,
-          commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
-    else:
-      padded_relative_embeddings = relative_embeddings
-    used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position]
-    return used_relative_embeddings
-
-  def _relative_position_to_absolute_position(self, x):
-    """
+        ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
+        return ret
+
+    def _get_relative_embeddings(self, relative_embeddings, length):
+        max_relative_position = 2 * self.window_size + 1
+        # Pad first before slice to avoid using cond ops.
+        pad_length = max(length - (self.window_size + 1), 0)
+        slice_start_position = max((self.window_size + 1) - length, 0)
+        slice_end_position = slice_start_position + 2 * length - 1
+        if pad_length > 0:
+            padded_relative_embeddings = F.pad(
+                relative_embeddings,
+                commons.convert_pad_shape([[0, 0], [pad_length, pad_length],
+                                           [0, 0]]))
+        else:
+            padded_relative_embeddings = relative_embeddings
+        used_relative_embeddings = padded_relative_embeddings[:,
+                                                              slice_start_position:
+                                                              slice_end_position]
+        return used_relative_embeddings
+
+    def _relative_position_to_absolute_position(self, x):
+        """
     x: [b, h, l, 2*l-1]
     ret: [b, h, l, l]
     """
-    batch, heads, length, _ = x.size()
-    # Concat columns of pad to shift from relative to absolute indexing.
-    x = F.pad(x, commons.convert_pad_shape([[0,0],[0,0],[0,0],[0,1]]))
-
-    # Concat extra elements so to add up to shape (len+1, 2*len-1).
-    x_flat = x.view([batch, heads, length * 2 * length])
-    x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]]))
-
-    # Reshape and slice out the padded elements.
-    x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:]
-    return x_final
-
-  def _absolute_position_to_relative_position(self, x):
-    """
+        batch, heads, length, _ = x.size()
+        # Concat columns of pad to shift from relative to absolute indexing.
+        x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0,
+                                                                         1]]))
+
+        # Concat extra elements so to add up to shape (len+1, 2*len-1).
+        x_flat = x.view([batch, heads, length * 2 * length])
+        x_flat = F.pad(
+            x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0,
+                                                                length - 1]]))
+
+        # Reshape and slice out the padded elements.
+        x_final = x_flat.view([batch, heads, length + 1,
+                               2 * length - 1])[:, :, :length, length - 1:]
+        return x_final
+
+    def _absolute_position_to_relative_position(self, x):
+        """
     x: [b, h, l, l]
     ret: [b, h, l, 2*l-1]
     """
-    batch, heads, length, _ = x.size()
-    # padd along column
-    x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]]))
-    x_flat = x.view([batch, heads, length**2 + length*(length -1)])
-    # add 0's in the beginning that will skew the elements after reshape
-    x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
-    x_final = x_flat.view([batch, heads, length, 2*length])[:,:,:,1:]
-    return x_final
-
-  def _attention_bias_proximal(self, length):
-    """Bias for self-attention to encourage attention to close positions.
+        batch, heads, length, _ = x.size()
+        # padd along column
+        x = F.pad(
+            x,
+            commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0,
+                                                                length - 1]]))
+        x_flat = x.view([batch, heads, length**2 + length * (length - 1)])
+        # add 0's in the beginning that will skew the elements after reshape
+        x_flat = F.pad(
+            x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
+        x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
+        return x_final
+
+    def _attention_bias_proximal(self, length):
+        """Bias for self-attention to encourage attention to close positions.
     Args:
       length: an integer scalar.
     Returns:
       a Tensor with shape [1, 1, length, length]
     """
-    r = torch.arange(length, dtype=torch.float32)
-    diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
-    return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
+        r = torch.arange(length, dtype=torch.float32)
+        diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
+        return torch.unsqueeze(
+            torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
 
 
 class FFN(nn.Module):
-  def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False):
-    super().__init__()
-    self.in_channels = in_channels
-    self.out_channels = out_channels
-    self.filter_channels = filter_channels
-    self.kernel_size = kernel_size
-    self.p_dropout = p_dropout
-    self.activation = activation
-    self.causal = causal
-
-    if causal:
-      self.padding = self._causal_padding
-    else:
-      self.padding = self._same_padding
-
-    self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
-    self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
-    self.drop = nn.Dropout(p_dropout)
-
-  def forward(self, x, x_mask):
-    x = self.conv_1(self.padding(x * x_mask))
-    if self.activation == "gelu":
-      x = x * torch.sigmoid(1.702 * x)
-    else:
-      x = torch.relu(x)
-    x = self.drop(x)
-    x = self.conv_2(self.padding(x * x_mask))
-    return x * x_mask
-  
-  def _causal_padding(self, x):
-    if self.kernel_size == 1:
-      return x
-    pad_l = self.kernel_size - 1
-    pad_r = 0
-    padding = [[0, 0], [0, 0], [pad_l, pad_r]]
-    x = F.pad(x, commons.convert_pad_shape(padding))
-    return x
-
-  def _same_padding(self, x):
-    if self.kernel_size == 1:
-      return x
-    pad_l = (self.kernel_size - 1) // 2
-    pad_r = self.kernel_size // 2
-    padding = [[0, 0], [0, 0], [pad_l, pad_r]]
-    x = F.pad(x, commons.convert_pad_shape(padding))
-    return x
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 filter_channels,
+                 kernel_size,
+                 p_dropout=0.,
+                 activation=None,
+                 causal=False):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.activation = activation
+        self.causal = causal
+
+        if causal:
+            self.padding = self._causal_padding
+        else:
+            self.padding = self._same_padding
+
+        self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
+        self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
+        self.drop = nn.Dropout(p_dropout)
+
+    def forward(self, x, x_mask):
+        x = self.conv_1(self.padding(x * x_mask))
+        if self.activation == "gelu":
+            x = x * torch.sigmoid(1.702 * x)
+        else:
+            x = torch.relu(x)
+        x = self.drop(x)
+        x = self.conv_2(self.padding(x * x_mask))
+        return x * x_mask
+
+    def _causal_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = self.kernel_size - 1
+        pad_r = 0
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x
+
+    def _same_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = (self.kernel_size - 1) // 2
+        pad_r = self.kernel_size // 2
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x

checkpoints/Default/config.json

@@ -0,0 +1,54 @@
+{
+  "train": {
+    "log_interval": 200,
+    "eval_interval": 1000,
+    "seed": 1234,
+    "epochs": 10000,
+    "learning_rate": 2e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 32,
+    "fp16_run": true,
+    "lr_decay": 0.999875,
+    "segment_size": 8192,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "training_files":"/www/training/dataset/train_with_paimeng2.txt",
+    "validation_files":"/www/training/dataset/val_filelist.txt",
+    "text_cleaners":["cjke_cleaners"],
+    "max_wav_value": 32768.0,
+    "sampling_rate": 22050,
+    "filter_length": 1024,
+    "hop_length": 256,
+    "win_length": 1024,
+    "n_mel_channels": 80,
+    "mel_fmin": 0.0,
+    "mel_fmax": null,
+    "add_blank": true,
+    "n_speakers": 50,
+    "cleaned_text": true
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0.1,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [8,8,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4],
+    "n_layers_q": 3,
+    "use_spectral_norm": false,
+    "gin_channels": 256
+  },
+  "symbols": ["_", ",", ".", "!", "?", "-", "~", "\u2026", "A", "E", "I", "N", "O", "Q", "U", "a", "b", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "r", "s", "t", "u", "v", "w", "y", "z", "\u0283", "\u02a7", "\u02a6", "\u026f", "\u0279", "\u0259", "\u0265", "\u207c", "\u02b0", "`", "\u2192", "\u2193", "\u2191", " "]
+}

checkpoints/Default/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6008ba1612a7e6fbefbdd633d07d6e8db07bebf6bcf1a4bb803e1dff636c5fcb
+size 120734883

checkpoints/NIjigasaki/config.json

@@ -0,0 +1,54 @@
+{
+  "train": {
+    "log_interval": 200,
+    "eval_interval": 1000,
+    "seed": 1234,
+    "epochs": 10000,
+    "learning_rate": 2e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 32,
+    "fp16_run": true,
+    "lr_decay": 0.999875,
+    "segment_size": 8192,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "training_files":"/www/training/dataset/train_with_paimeng2.txt",
+    "validation_files":"/www/training/dataset/val_filelist.txt",
+    "text_cleaners":["cjke_cleaners"],
+    "max_wav_value": 32768.0,
+    "sampling_rate": 22050,
+    "filter_length": 1024,
+    "hop_length": 256,
+    "win_length": 1024,
+    "n_mel_channels": 80,
+    "mel_fmin": 0.0,
+    "mel_fmax": null,
+    "add_blank": true,
+    "n_speakers": 50,
+    "cleaned_text": true
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0.1,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [8,8,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4],
+    "n_layers_q": 3,
+    "use_spectral_norm": false,
+    "gin_channels": 256
+  },
+  "symbols": ["_", ",", ".", "!", "?", "-", "~", "\u2026", "A", "E", "I", "N", "O", "Q", "U", "a", "b", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "r", "s", "t", "u", "v", "w", "y", "z", "\u0283", "\u02a7", "\u02a6", "\u026f", "\u0279", "\u0259", "\u0265", "\u207c", "\u02b0", "`", "\u2192", "\u2193", "\u2191", " "]
+}

checkpoints/NIjigasaki/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcdfabd68e081f0b9b0b2ac7600fd6d6124102607718680fcd8611cee9d5a2da
+size 120734883

checkpoints/ShojoKageki/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6008ba1612a7e6fbefbdd633d07d6e8db07bebf6bcf1a4bb803e1dff636c5fcb
+size 120734883

checkpoints/Starlight/config.json

@@ -0,0 +1,54 @@
+{
+  "train": {
+    "log_interval": 200,
+    "eval_interval": 1000,
+    "seed": 1234,
+    "epochs": 10000,
+    "learning_rate": 2e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 32,
+    "fp16_run": true,
+    "lr_decay": 0.999875,
+    "segment_size": 8192,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "training_files":"/www/training/dataset/train_with_paimeng2.txt",
+    "validation_files":"/www/training/dataset/val_filelist.txt",
+    "text_cleaners":["cjke_cleaners"],
+    "max_wav_value": 32768.0,
+    "sampling_rate": 22050,
+    "filter_length": 1024,
+    "hop_length": 256,
+    "win_length": 1024,
+    "n_mel_channels": 80,
+    "mel_fmin": 0.0,
+    "mel_fmax": null,
+    "add_blank": true,
+    "n_speakers": 50,
+    "cleaned_text": true
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0.1,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [8,8,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4],
+    "n_layers_q": 3,
+    "use_spectral_norm": false,
+    "gin_channels": 256
+  },
+  "symbols": ["_", ",", ".", "!", "?", "-", "~", "\u2026", "A", "E", "I", "N", "O", "Q", "U", "a", "b", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "r", "s", "t", "u", "v", "w", "y", "z", "\u0283", "\u02a7", "\u02a6", "\u026f", "\u0279", "\u0259", "\u0265", "\u207c", "\u02b0", "`", "\u2192", "\u2193", "\u2191", " "]
+}

checkpoints/Starlight/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3657d9607364481af521ce67ca1d6a3d3e710b28dca7d2c2bbe19f44b3b67e4a
+size 120734883

checkpoints/info.json

@@ -167,10 +167,10 @@
                 "name": "高咲侑"
             }
         },
-        "checkpoint":  "checkpoints/tmp/model.pth"    
+        "checkpoint":  "checkpoints/Default/model.onnx"    
         
     },
-    "Seisho-betterchinese":{
+    "ShojoKageki":{
         "speakers":{
             "華恋":{
                 "sid": 21,
@@ -288,7 +288,7 @@
                 "name": "墨小菊"
             }
         },
-        "checkpoint": "checkpoints/ShojoKageki/model.pth"   
+        "checkpoint": "checkpoints/ShojoKageki/model.onnx"   
     },
     "Nijigasaki":{
         "speakers":{
@@ -353,72 +353,6 @@
                 "name": "高咲侑"
             }
         },
-        "checkpoint":  "checkpoints/paimeng/model.pth"   
-    },
-    "Nijigasaki-biaobei":{
-        "speakers":{
-            "歩夢":{
-                "sid": 1,
-                "speech": "みなさん、はじめまして。上原歩夢です。",
-                "name": "歩夢"
-            },
-            "かすみ":{
-                "sid": 2,
-                "speech": "みんなのアイドルかすみんだよー。",
-                "name": "かすみ"
-            },
-            "しずく":{
-                "sid": 3,
-                "speech": "みなさん、こんにちは。しずくです。",
-                "name": "しずく"
-            },
-            "果林":{
-                "sid": 4,
-                "speech": "ハーイ。　朝香果林よ。よろしくね",
-                "name": "果林"
-            },
-            "愛":{
-                "sid": 5,
-                "speech": "ちっすー。アタシは愛。",
-                "name": "愛"
-            },
-            "せつ菜":{
-                "sid": 7,
-                "speech": "絶えぬ命は，常世に在らず。終わらぬ芝居も，夢幻のごとく。儚く燃えゆく，さだめであれば。舞台に刻まん，刹那の瞬き。",
-                "name": "せつ菜"
-            },
-            "エマ":{
-                "sid": 8,
-                "speech": "こんにちは、エマです。自然溢れるスイスからやってきましたっ。",
-                "name": "エマ"
-            },
-            "璃奈":{
-                "sid": 9,
-                "speech": "私、天王寺璃奈。とってもきゅーとな女の子。ホントだよ？",
-                "name": "璃奈"
-            },
-            "栞子":{
-                "sid": 10,
-                "speech": "みなさん、初めまして。三船栞子と申します。",
-                "name": "栞子"
-            },
-            "ランジュ":{
-                "sid": 11,
-                "speech": "你好啊，我是钟岚珠。",
-                "name": "ランジュ"
-            },
-            "ミア":{
-                "sid": 12,
-                "speech": "ボクはミア・テイラー。",
-                "name": "ミア"
-            },
-            "高咲侑":{
-                "sid": 0, 
-                "speech": "只选一个做不到啊",
-                "name": "高咲侑"
-            }
-        },
-        "checkpoint":  "checkpoints/biaobei/model.pth"   
+        "checkpoint":  "checkpoints/NIjigasaki/model.onnx"   
     }
-
 }

cleaners/JapaneseCleaner.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a659eb68d12d4a88ef7dfde6086b9974cd4d43634f7e4bfe710d5537cdd61a75
+size 3097600

cleaners/char.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:888ee94c5a8a7a26d24ab3f1b7155441351954fd51ea06b4a2f78bd742492b2f
+size 262496

cleaners/matrix.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62fd16b4f64c851d5dc352ef0d5740c5fc83ddc7c203b2b0b1fc5271969a14ce
+size 3792262

cleaners/sys.dic

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca57d9029691a70a5dfb99afc2844180256161d7130da65b1a867510e129b9a6
+size 103073776

commons.py

@@ -1,97 +1,161 @@
 import math
+
 import torch
 from torch.nn import functional as F
-import torch.jit
 
 
-def script_method(fn, _rcb=None):
-  return fn
+def init_weights(m, mean=0.0, std=0.01):
+    classname = m.__class__.__name__
+    if classname.find("Conv") != -1:
+        m.weight.data.normal_(mean, std)
 
 
-def script(obj, optimize=True, _frames_up=0, _rcb=None):
-  return obj
+def get_padding(kernel_size, dilation=1):
+    return int((kernel_size * dilation - dilation) / 2)
 
 
-torch.jit.script_method = script_method
-torch.jit.script = script
+def convert_pad_shape(pad_shape):
+    pad_shape = [item for sublist in reversed(pad_shape) for item in sublist]
+    return pad_shape
 
 
-def init_weights(m, mean=0.0, std=0.01):
-  classname = m.__class__.__name__
-  if classname.find("Conv") != -1:
-    m.weight.data.normal_(mean, std)
+def intersperse(lst, item):
+    result = [item] * (len(lst) * 2 + 1)
+    result[1::2] = lst
+    return result
 
 
-def get_padding(kernel_size, dilation=1):
-  return int((kernel_size*dilation - dilation)/2)
+def kl_divergence(m_p, logs_p, m_q, logs_q):
+    """KL(P||Q)"""
+    kl = (logs_q - logs_p) - 0.5
+    kl += 0.5 * (torch.exp(2. * logs_p) +
+                 ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
+    return kl
 
 
-def intersperse(lst, item):
-  result = [item] * (len(lst) * 2 + 1)
-  result[1::2] = lst
-  return result
+def rand_gumbel(shape):
+    """Sample from the Gumbel distribution, protect from overflows."""
+    uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
+    return -torch.log(-torch.log(uniform_samples))
+
+
+def rand_gumbel_like(x):
+    g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
+    return g
 
 
 def slice_segments(x, ids_str, segment_size=4):
-  ret = torch.zeros_like(x[:, :, :segment_size])
-  for i in range(x.size(0)):
-    idx_str = ids_str[i]
-    idx_end = idx_str + segment_size
-    ret[i] = x[i, :, idx_str:idx_end]
-  return ret
+    ret = torch.zeros_like(x[:, :, :segment_size])
+    for i in range(x.size(0)):
+        idx_str = ids_str[i]
+        idx_end = idx_str + segment_size
+        ret[i] = x[i, :, idx_str:idx_end]
+    return ret
 
 
 def rand_slice_segments(x, x_lengths=None, segment_size=4):
-  b, d, t = x.size()
-  if x_lengths is None:
-    x_lengths = t
-  ids_str_max = x_lengths - segment_size + 1
-  ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
-  ret = slice_segments(x, ids_str, segment_size)
-  return ret, ids_str
+    b, d, t = x.size()
+    if x_lengths is None:
+        x_lengths = t
+    ids_str_max = x_lengths - segment_size + 1
+    ids_str = (torch.rand([b]).to(device=x.device) *
+               ids_str_max).to(dtype=torch.long)
+    ret = slice_segments(x, ids_str, segment_size)
+    return ret, ids_str
+
+
+def get_timing_signal_1d(length,
+                         channels,
+                         min_timescale=1.0,
+                         max_timescale=1.0e4):
+    position = torch.arange(length, dtype=torch.float)
+    num_timescales = channels // 2
+    log_timescale_increment = (
+        math.log(float(max_timescale) / float(min_timescale)) /
+        (num_timescales - 1))
+    inv_timescales = min_timescale * torch.exp(
+        torch.arange(num_timescales, dtype=torch.float) *
+        -log_timescale_increment)
+    scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
+    signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
+    signal = F.pad(signal, [0, 0, 0, channels % 2])
+    signal = signal.view(1, channels, length)
+    return signal
+
+
+def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
+    b, channels, length = x.size()
+    signal = get_timing_signal_1d(length, channels, min_timescale,
+                                  max_timescale)
+    return x + signal.to(dtype=x.dtype, device=x.device)
+
+
+def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
+    b, channels, length = x.size()
+    signal = get_timing_signal_1d(length, channels, min_timescale,
+                                  max_timescale)
+    return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
 
 
 def subsequent_mask(length):
-  mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
-  return mask
+    mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
+    return mask
 
 
 @torch.jit.script
 def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
-  n_channels_int = n_channels[0]
-  in_act = input_a + input_b
-  t_act = torch.tanh(in_act[:, :n_channels_int, :])
-  s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
-  acts = t_act * s_act
-  return acts
+    n_channels_int = n_channels[0]
+    in_act = input_a + input_b
+    t_act = torch.tanh(in_act[:, :n_channels_int, :])
+    s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+    acts = t_act * s_act
+    return acts
 
 
-def convert_pad_shape(pad_shape):
-  l = pad_shape[::-1]
-  pad_shape = [item for sublist in l for item in sublist]
-  return pad_shape
+def shift_1d(x):
+    x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
+    return x
 
 
 def sequence_mask(length, max_length=None):
-  if max_length is None:
-    max_length = length.max()
-  x = torch.arange(max_length, dtype=length.dtype, device=length.device)
-  return x.unsqueeze(0) < length.unsqueeze(1)
+    if max_length is None:
+        max_length = length.max()
+    x = torch.arange(max_length, dtype=length.dtype, device=length.device)
+    return x.unsqueeze(0) < length.unsqueeze(1)
 
 
 def generate_path(duration, mask):
-  """
+    """
   duration: [b, 1, t_x]
   mask: [b, 1, t_y, t_x]
   """
-  device = duration.device
-  
-  b, _, t_y, t_x = mask.shape
-  cum_duration = torch.cumsum(duration, -1)
-  
-  cum_duration_flat = cum_duration.view(b * t_x)
-  path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
-  path = path.view(b, t_x, t_y)
-  path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
-  path = path.unsqueeze(1).transpose(2,3) * mask
-  return path
+    device = duration.device
+
+    b, _, t_y, t_x = mask.shape
+    cum_duration = torch.cumsum(duration, -1)
+
+    cum_duration_flat = cum_duration.view(b * t_x)
+    path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
+    path = path.view(b, t_x, t_y)
+    path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]
+                                                 ]))[:, :-1]
+    path = path.unsqueeze(1).transpose(2, 3) * mask
+    return path
+
+
+def clip_grad_value_(parameters, clip_value, norm_type=2):
+    if isinstance(parameters, torch.Tensor):
+        parameters = [parameters]
+    parameters = list(filter(lambda p: p.grad is not None, parameters))
+    norm_type = float(norm_type)
+    if clip_value is not None:
+        clip_value = float(clip_value)
+
+    total_norm = 0
+    for p in parameters:
+        param_norm = p.grad.data.norm(norm_type)
+        total_norm += param_norm.item()**norm_type
+        if clip_value is not None:
+            p.grad.data.clamp_(min=-clip_value, max=clip_value)
+    total_norm = total_norm**(1. / norm_type)
+    return total_norm

data_utils.py

@@ -0,0 +1,307 @@
+import os
+import random
+
+import torch
+import torchaudio
+import torch.utils.data
+
+import commons
+from mel_processing import spectrogram_torch
+from utils import load_filepaths_and_text
+
+
+class TextAudioSpeakerLoader(torch.utils.data.Dataset):
+    """
+        1) loads audio, speaker_id, text pairs
+        2) normalizes text and converts them to sequences of integers
+        3) computes spectrograms from audio files.
+    """
+    def __init__(self, audiopaths_sid_text, hparams):
+        self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text)
+        # self.text_cleaners = hparams.text_cleaners
+        self.max_wav_value = hparams.max_wav_value
+        self.sampling_rate = hparams.sampling_rate
+        self.filter_length = hparams.filter_length
+        self.hop_length = hparams.hop_length
+        self.win_length = hparams.win_length
+        self.sampling_rate = hparams.sampling_rate
+        self.src_sampling_rate = getattr(hparams, "src_sampling_rate",
+                                         self.sampling_rate)
+
+        self.cleaned_text = getattr(hparams, "cleaned_text", False)
+
+        self.add_blank = hparams.add_blank
+        self.min_text_len = getattr(hparams, "min_text_len", 1)
+        self.max_text_len = getattr(hparams, "max_text_len", 190)
+
+        phone_file = getattr(hparams, "phone_table", None)
+        self.phone_dict = None
+        if phone_file is not None:
+            self.phone_dict = {}
+            with open(phone_file) as fin:
+                for line in fin:
+                    arr = line.strip().split()
+                    self.phone_dict[arr[0]] = int(arr[1])
+
+        speaker_file = getattr(hparams, "speaker_table", None)
+        self.speaker_dict = None
+        if speaker_file is not None:
+            self.speaker_dict = {}
+            with open(speaker_file) as fin:
+                for line in fin:
+                    arr = line.strip().split()
+                    self.speaker_dict[arr[0]] = int(arr[1])
+
+        random.seed(1234)
+        random.shuffle(self.audiopaths_sid_text)
+        self._filter()
+
+    def _filter(self):
+        """
+        Filter text & store spec lengths
+        """
+        # Store spectrogram lengths for Bucketing
+        # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
+        # spec_length = wav_length // hop_length
+
+        audiopaths_sid_text_new = []
+        lengths = []
+        for item in self.audiopaths_sid_text:
+            audiopath = item[0]
+            # filename|text or filename|speaker|text
+            text = item[1] if len(item) == 2 else item[2]
+            if self.min_text_len <= len(text) and len(
+                    text) <= self.max_text_len:
+                audiopaths_sid_text_new.append(item)
+                lengths.append(
+                    int(
+                        os.path.getsize(audiopath) * self.sampling_rate /
+                        self.src_sampling_rate) // (2 * self.hop_length))
+        self.audiopaths_sid_text = audiopaths_sid_text_new
+        self.lengths = lengths
+
+    def get_audio_text_speaker_pair(self, audiopath_sid_text):
+        audiopath = audiopath_sid_text[0]
+        if len(audiopath_sid_text) == 2:  # filename|text
+            sid = 0
+            text = audiopath_sid_text[1]
+        else:  # filename|speaker|text
+            sid = self.speaker_dict[audiopath_sid_text[1]]
+            text = audiopath_sid_text[2]
+        text = self.get_text(text)
+        spec, wav = self.get_audio(audiopath)
+        sid = self.get_sid(sid)
+        return (text, spec, wav, sid)
+
+    def get_audio(self, filename):
+        audio, sampling_rate = torchaudio.load(filename, normalize=False)
+        if sampling_rate != self.sampling_rate:
+            audio = audio.to(torch.float)
+            audio = torchaudio.transforms.Resample(sampling_rate,
+                                                   self.sampling_rate)(audio)
+            audio = audio.to(torch.int16)
+        audio = audio[0]  # Get the first channel
+        audio_norm = audio / self.max_wav_value
+        audio_norm = audio_norm.unsqueeze(0)
+        spec = spectrogram_torch(audio_norm,
+                                 self.filter_length,
+                                 self.sampling_rate,
+                                 self.hop_length,
+                                 self.win_length,
+                                 center=False)
+        spec = torch.squeeze(spec, 0)
+        return spec, audio_norm
+
+    def get_text(self, text):
+        text_norm = [self.phone_dict[phone] for phone in text.split()]
+        if self.add_blank:
+            text_norm = commons.intersperse(text_norm, 0)
+        text_norm = torch.LongTensor(text_norm)
+        return text_norm
+
+    def get_sid(self, sid):
+        sid = torch.LongTensor([int(sid)])
+        return sid
+
+    def __getitem__(self, index):
+        return self.get_audio_text_speaker_pair(
+            self.audiopaths_sid_text[index])
+
+    def __len__(self):
+        return len(self.audiopaths_sid_text)
+
+
+class TextAudioSpeakerCollate():
+    """ Zero-pads model inputs and targets
+    """
+    def __init__(self, return_ids=False):
+        self.return_ids = return_ids
+
+    def __call__(self, batch):
+        """Collate's training batch from normalized text, audio and speaker identities
+        PARAMS
+        ------
+        batch: [text_normalized, spec_normalized, wav_normalized, sid]
+        """
+        # Right zero-pad all one-hot text sequences to max input length
+        _, ids_sorted_decreasing = torch.sort(torch.LongTensor(
+            [x[1].size(1) for x in batch]),
+            dim=0,
+            descending=True)
+
+        max_text_len = max([len(x[0]) for x in batch])
+        max_spec_len = max([x[1].size(1) for x in batch])
+        max_wav_len = max([x[2].size(1) for x in batch])
+
+        text_lengths = torch.LongTensor(len(batch))
+        spec_lengths = torch.LongTensor(len(batch))
+        wav_lengths = torch.LongTensor(len(batch))
+        sid = torch.LongTensor(len(batch))
+
+        text_padded = torch.LongTensor(len(batch), max_text_len)
+        spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0),
+                                        max_spec_len)
+        wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
+        text_padded.zero_()
+        spec_padded.zero_()
+        wav_padded.zero_()
+        for i in range(len(ids_sorted_decreasing)):
+            row = batch[ids_sorted_decreasing[i]]
+
+            text = row[0]
+            text_padded[i, :text.size(0)] = text
+            text_lengths[i] = text.size(0)
+
+            spec = row[1]
+            spec_padded[i, :, :spec.size(1)] = spec
+            spec_lengths[i] = spec.size(1)
+
+            wav = row[2]
+            wav_padded[i, :, :wav.size(1)] = wav
+            wav_lengths[i] = wav.size(1)
+
+            sid[i] = row[3]
+
+        if self.return_ids:
+            return (text_padded, text_lengths, spec_padded, spec_lengths,
+                    wav_padded, wav_lengths, sid, ids_sorted_decreasing)
+        return (text_padded, text_lengths, spec_padded, spec_lengths,
+                wav_padded, wav_lengths, sid)
+
+
+class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler
+                               ):
+    """
+    Maintain similar input lengths in a batch.
+    Length groups are specified by boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any batch is included either
+    {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
+
+    It removes samples which are not included in the boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1
+    or length(x) > b3 are discarded.
+    """
+    def __init__(self,
+                 dataset,
+                 batch_size,
+                 boundaries,
+                 num_replicas=None,
+                 rank=None,
+                 shuffle=True):
+        super().__init__(dataset,
+                         num_replicas=num_replicas,
+                         rank=rank,
+                         shuffle=shuffle)
+        self.lengths = dataset.lengths
+        self.batch_size = batch_size
+        self.boundaries = boundaries
+
+        self.buckets, self.num_samples_per_bucket = self._create_buckets()
+        self.total_size = sum(self.num_samples_per_bucket)
+        self.num_samples = self.total_size // self.num_replicas
+
+    def _create_buckets(self):
+        buckets = [[] for _ in range(len(self.boundaries) - 1)]
+        for i in range(len(self.lengths)):
+            length = self.lengths[i]
+            idx_bucket = self._bisect(length)
+            if idx_bucket != -1:
+                buckets[idx_bucket].append(i)
+
+        for i in range(len(buckets) - 1, 0, -1):
+            if len(buckets[i]) == 0:
+                buckets.pop(i)
+                self.boundaries.pop(i + 1)
+
+        num_samples_per_bucket = []
+        for i in range(len(buckets)):
+            len_bucket = len(buckets[i])
+            total_batch_size = self.num_replicas * self.batch_size
+            rem = (total_batch_size -
+                   (len_bucket % total_batch_size)) % total_batch_size
+            num_samples_per_bucket.append(len_bucket + rem)
+        return buckets, num_samples_per_bucket
+
+    def __iter__(self):
+        # deterministically shuffle based on epoch
+        g = torch.Generator()
+        g.manual_seed(self.epoch)
+
+        indices = []
+        if self.shuffle:
+            for bucket in self.buckets:
+                indices.append(
+                    torch.randperm(len(bucket), generator=g).tolist())
+        else:
+            for bucket in self.buckets:
+                indices.append(list(range(len(bucket))))
+
+        batches = []
+        for i in range(len(self.buckets)):
+            bucket = self.buckets[i]
+            len_bucket = len(bucket)
+            ids_bucket = indices[i]
+            num_samples_bucket = self.num_samples_per_bucket[i]
+
+            # add extra samples to make it evenly divisible
+            rem = num_samples_bucket - len_bucket
+            ids_bucket = ids_bucket + ids_bucket * (
+                rem // len_bucket) + ids_bucket[:(rem % len_bucket)]
+
+            # subsample
+            ids_bucket = ids_bucket[self.rank::self.num_replicas]
+
+            # batching
+            for j in range(len(ids_bucket) // self.batch_size):
+                batch = [
+                    bucket[idx]
+                    for idx in ids_bucket[j * self.batch_size:(j + 1) *
+                                          self.batch_size]
+                ]
+                batches.append(batch)
+
+        if self.shuffle:
+            batch_ids = torch.randperm(len(batches), generator=g).tolist()
+            batches = [batches[i] for i in batch_ids]
+        self.batches = batches
+
+        assert len(self.batches) * self.batch_size == self.num_samples
+        return iter(self.batches)
+
+    def _bisect(self, x, lo=0, hi=None):
+        if hi is None:
+            hi = len(self.boundaries) - 1
+
+        if hi > lo:
+            mid = (hi + lo) // 2
+            if self.boundaries[mid] < x and x <= self.boundaries[mid + 1]:
+                return mid
+            elif x <= self.boundaries[mid]:
+                return self._bisect(x, lo, mid)
+            else:
+                return self._bisect(x, mid + 1, hi)
+        else:
+            return -1
+
+    def __len__(self):
+        return self.num_samples // self.batch_size

export_onnx.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2022, Yongqiang Li (yongqiangli@alumni.hust.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+import sys
+
+import torch
+
+from models import SynthesizerTrn
+import utils
+
+try:
+    import onnxruntime as ort
+except ImportError:
+    print('Please install onnxruntime!')
+    sys.exit(1)
+
+
+def to_numpy(tensor):
+    return tensor.detach().cpu().numpy() if tensor.requires_grad \
+        else tensor.detach().numpy()
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='export onnx model')
+    parser.add_argument('--checkpoint', required=True, help='checkpoint')
+    parser.add_argument('--cfg', required=True, help='config file')
+    parser.add_argument('--onnx_model', required=True, help='onnx model name')
+    # parser.add_argument('--phone_table',
+    #                     required=True,
+    #                     help='input phone dict')
+    # parser.add_argument('--speaker_table', default=None, help='speaker table')
+    # parser.add_argument("--speaker_num", required=True,
+    #                     type=int, help="speaker num")
+    parser.add_argument(
+        '--providers',
+        required=False,
+        default='CPUExecutionProvider',
+        choices=['CUDAExecutionProvider', 'CPUExecutionProvider'],
+        help='the model to send request to')
+    args = parser.parse_args()
+    return args
+
+
+def get_data_from_cfg(cfg_path: str):
+    assert os.path.isfile(cfg_path)
+    with open(cfg_path, 'r') as f:
+        data = json.load(f)
+        symbols = data["symbols"]
+        speaker_num = data["data"]["n_speakers"]
+    return len(symbols), speaker_num
+
+
+def main():
+    args = get_args()
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+    hps = utils.get_hparams_from_file(args.cfg)
+    # with open(args.phone_table) as p_f:
+    #     phone_num = len(p_f.readlines()) + 1
+    # num_speakers = 1
+    # if args.speaker_table is not None:
+    #     num_speakers = len(open(args.speaker_table).readlines()) + 1
+    phone_num, num_speakers = get_data_from_cfg(args.cfg)
+    net_g = SynthesizerTrn(phone_num,
+                           hps.data.filter_length // 2 + 1,
+                           hps.train.segment_size // hps.data.hop_length,
+                           n_speakers=num_speakers,
+                           **hps.model)
+    utils.load_checkpoint(args.checkpoint, net_g, None)
+    net_g.forward = net_g.export_forward
+    net_g.eval()
+
+    seq = torch.randint(low=0, high=phone_num, size=(1, 10), dtype=torch.long)
+    seq_len = torch.IntTensor([seq.size(1)]).long()
+
+    # noise(可用于控制感情等变化程度) lenth(可用于控制整体语速) noisew(控制音素发音长度变化程度)
+    # 参考 https://github.com/gbxh/genshinTTS
+    scales = torch.FloatTensor([0.667, 1.0, 0.8])
+    # make triton dynamic shape happy
+    scales = scales.unsqueeze(0)
+    sid = torch.IntTensor([0]).long()
+
+    dummy_input = (seq, seq_len, scales, sid)
+    torch.onnx.export(model=net_g,
+                      args=dummy_input,
+                      f=args.onnx_model,
+                      input_names=['input', 'input_lengths', 'scales', 'sid'],
+                      output_names=['output'],
+                      dynamic_axes={
+                          'input': {
+                              0: 'batch',
+                              1: 'phonemes'
+                          },
+                          'input_lengths': {
+                              0: 'batch'
+                          },
+                          'scales': {
+                              0: 'batch'
+                          },
+                          'sid': {
+                              0: 'batch'
+                          },
+                          'output': {
+                              0: 'batch',
+                              1: 'audio',
+                              2: 'audio_length'
+                          }
+                      },
+                      opset_version=13,
+                      verbose=False)
+
+    # Verify onnx precision
+    torch_output = net_g(seq, seq_len, scales, sid)
+    providers = [args.providers]
+    ort_sess = ort.InferenceSession(args.onnx_model, providers=providers)
+    ort_inputs = {
+        'input': to_numpy(seq),
+        'input_lengths': to_numpy(seq_len),
+        'scales': to_numpy(scales),
+        'sid': to_numpy(sid),
+    }
+    onnx_output = ort_sess.run(None, ort_inputs)
+
+
+if __name__ == '__main__':
+    main()

inference.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2022, Yongqiang Li (yongqiangli@alumni.hust.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+
+import numpy as np
+from scipy.io import wavfile
+import torch
+
+import commons
+from models import SynthesizerTrn
+import utils
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='inference')
+    parser.add_argument('--checkpoint', required=True, help='checkpoint')
+    parser.add_argument('--cfg', required=True, help='config file')
+    parser.add_argument('--outdir', required=True, help='ouput directory')
+    parser.add_argument('--phone_table',
+                        required=True,
+                        help='input phone dict')
+    parser.add_argument('--speaker_table', default=None, help='speaker table')
+    parser.add_argument('--test_file', required=True, help='test file')
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+    print(args)
+    phone_dict = {}
+    with open(args.phone_table) as p_f:
+        for line in p_f:
+            phone_id = line.strip().split()
+            phone_dict[phone_id[0]] = int(phone_id[1])
+    speaker_dict = {}
+    if args.speaker_table is not None:
+        with open(args.speaker_table) as p_f:
+            for line in p_f:
+                arr = line.strip().split()
+                assert len(arr) == 2
+                speaker_dict[arr[0]] = int(arr[1])
+    hps = utils.get_hparams_from_file(args.cfg)
+
+    net_g = SynthesizerTrn(
+        len(phone_dict) + 1,
+        hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        n_speakers=len(speaker_dict) + 1,  # 0 is kept for unknown speaker
+        **hps.model).cuda()
+    net_g.eval()
+    utils.load_checkpoint(args.checkpoint, net_g, None)
+
+    with open(args.test_file) as fin:
+        for line in fin:
+            arr = line.strip().split("|")
+            audio_path = arr[0]
+            if len(arr) == 2:
+                sid = 0
+                text = arr[1]
+            else:
+                sid = speaker_dict[arr[1]]
+                text = arr[2]
+            seq = [phone_dict[symbol] for symbol in text.split()]
+            if hps.data.add_blank:
+                seq = commons.intersperse(seq, 0)
+            seq = torch.LongTensor(seq)
+            with torch.no_grad():
+                x = seq.cuda().unsqueeze(0)
+                x_length = torch.LongTensor([seq.size(0)]).cuda()
+                sid = torch.LongTensor([sid]).cuda()
+                audio = net_g.infer(
+                    x,
+                    x_length,
+                    sid=sid,
+                    noise_scale=.667,
+                    noise_scale_w=0.8,
+                    length_scale=1)[0][0, 0].data.cpu().float().numpy()
+                audio *= 32767 / max(0.01, np.max(np.abs(audio))) * 0.6
+                audio = np.clip(audio, -32767.0, 32767.0)
+                wavfile.write(args.outdir + "/" + audio_path.split("/")[-1],
+                              hps.data.sampling_rate, audio.astype(np.int16))
+
+
+if __name__ == '__main__':
+    main()

inference_onnx.py

@@ -0,0 +1,148 @@
+# Copyright (c) 2022, Yongqiang Li (yongqiangli@alumni.hust.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from text import text_to_sequence
+import numpy as np
+from scipy.io import wavfile
+import torch
+import json
+import commons
+import utils
+import sys
+import pathlib
+
+try:
+    import onnxruntime as ort
+except ImportError:
+    print('Please install onnxruntime!')
+    sys.exit(1)
+
+
+def to_numpy(tensor: torch.Tensor):
+    return tensor.detach().cpu().numpy() if tensor.requires_grad \
+        else tensor.detach().numpy()
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='inference')
+    parser.add_argument('--onnx_model', required=True, help='onnx model')
+    parser.add_argument('--cfg', required=True, help='config file')
+    parser.add_argument('--outdir', default="onnx_output",
+                        help='ouput directory')
+    # parser.add_argument('--phone_table',
+    #                     required=True,
+    #                     help='input phone dict')
+    # parser.add_argument('--speaker_table', default=None, help='speaker table')
+    parser.add_argument('--test_file', required=True, help='test file')
+    args = parser.parse_args()
+    return args
+
+
+def get_symbols_from_json(path):
+    import os
+    assert os.path.isfile(path)
+    with open(path, 'r') as f:
+        data = json.load(f)
+    return data['symbols']
+
+
+def main():
+    args = get_args()
+    print(args)
+    if not pathlib.Path(args.outdir).exists():
+        pathlib.Path(args.outdir).mkdir(exist_ok=True, parents=True)
+    # phones =
+    symbols = get_symbols_from_json(args.cfg)
+    phone_dict = {
+        symbol: i for i, symbol in enumerate(symbols)
+    }
+
+    # speaker_dict = {}
+    # if args.speaker_table is not None:
+    #     with open(args.speaker_table) as p_f:
+    #         for line in p_f:
+    #             arr = line.strip().split()
+    #             assert len(arr) == 2
+    #             speaker_dict[arr[0]] = int(arr[1])
+    hps = utils.get_hparams_from_file(args.cfg)
+
+    ort_sess = ort.InferenceSession(args.onnx_model)
+
+    with open(args.test_file) as fin:
+        for line in fin:
+            arr = line.strip().split("|")
+            audio_path = arr[0]
+        
+            # TODO: 控制说话人编号
+            sid = 3
+            text = '[ZH]你好,重庆市位于四川省东边[ZH]'
+            # else:
+            #     sid = speaker_dict[arr[1]]
+            #     text = arr[2]
+            seq = text_to_sequence(text, cleaner_names=hps.data.text_cleaners
+                                   )
+            if hps.data.add_blank:
+                seq = commons.intersperse(seq, 0)
+
+            # if hps.data.add_blank:
+            #     seq = commons.intersperse(seq, 0)
+            with torch.no_grad():
+                #         x = torch.LongTensor([seq])
+                #         x_len = torch.IntTensor([x.size(1)]).long()
+                #         sid = torch.LongTensor([sid]).long()
+                #         scales = torch.FloatTensor([0.667, 1.0, 1])
+                # # make triton dynamic shape happy
+                #         scales = scales.unsqueeze(0)
+
+                # use numpy to replace torch
+                x = np.array([seq], dtype=np.int64)
+                x_len = np.array([x.shape[1]], dtype=np.int64)
+                sid = np.array([sid], dtype=np.int64)
+                # noise(可用于控制感情等变化程度) lenth(可用于控制整体语速) noisew(控制音素发音长度变化程度)
+                # 参考 https://github.com/gbxh/genshinTTS
+                scales = np.array([0.667, 0.8, 1], dtype=np.float32)
+                # scales = scales[np.newaxis, :]
+                # scales.reshape(1, -1)
+                scales.resize(1, 3)
+
+                ort_inputs = {
+                    'input': x,
+                    'input_lengths': x_len,
+                    'scales': scales,
+                    'sid': sid
+                }
+
+                # ort_inputs = {
+                #     'input': to_numpy(x),
+                #     'input_lengths': to_numpy(x_len),
+                #     'scales': to_numpy(scales),
+                #     'sid': to_numpy(sid)
+                # }
+                import time
+                # start_time = time.time()
+                start_time = time.perf_counter()
+                audio = np.squeeze(ort_sess.run(None, ort_inputs))
+                audio *= 32767.0 / max(0.01, np.max(np.abs(audio))) * 0.6
+                audio = np.clip(audio, -32767.0, 32767.0)
+                end_time = time.perf_counter()
+                # end_time = time.time()
+                print("infer time cost: ", end_time - start_time, "s")
+
+                wavfile.write(args.outdir + "/" + audio_path.split("/")[-1],
+                              hps.data.sampling_rate, audio.astype(np.int16))
+
+
+if __name__ == '__main__':
+    main()

local_run.py

@@ -0,0 +1,137 @@
+import argparse
+from text import text_to_sequence
+import numpy as np
+from scipy.io import wavfile
+import torch
+import json
+import commons
+import utils
+import sys
+import pathlib
+from flask import Flask, request
+import threading
+import onnxruntime as ort
+import time
+from pydub import AudioSegment
+import io
+import os
+from transformers import AutoTokenizer, AutoModel
+import tkinter as tk
+from tkinter import scrolledtext
+from scipy.io.wavfile import write
+def get_args():
+    parser = argparse.ArgumentParser(description='inference')
+    parser.add_argument('--onnx_model', default = './moe/model.onnx')
+    parser.add_argument('--cfg', default="./moe/config_v.json")
+    parser.add_argument('--outdir', default="./moe",
+                        help='ouput folder')
+    parser.add_argument('--audio',
+                    type=str,
+                    help='你要替换的音频文件的,假设这些音频文件为temp1、temp2、temp3......',
+                    default = 'D:/app_develop/live2d_whole/2010002/sounds/temp.wav')
+    parser.add_argument('--ChatGLM',default = "./moe",
+                        help='https://github.com/THUDM/ChatGLM-6B')
+    args = parser.parse_args()
+    return args
+
+def to_numpy(tensor: torch.Tensor):
+    return tensor.detach().cpu().numpy() if tensor.requires_grad \
+        else tensor.detach().numpy()
+
+def get_symbols_from_json(path):
+    import os
+    assert os.path.isfile(path)
+    with open(path, 'r') as f:
+        data = json.load(f)
+    return data['symbols']
+
+args = get_args()
+symbols = get_symbols_from_json(args.cfg)
+phone_dict = {
+        symbol: i for i, symbol in enumerate(symbols)
+    }
+hps = utils.get_hparams_from_file(args.cfg)
+ort_sess = ort.InferenceSession(args.onnx_model)
+
+def is_japanese(string):
+        for ch in string:
+            if ord(ch) > 0x3040 and ord(ch) < 0x30FF:
+                return True
+        return False 
+
+def infer(text):
+    #选择你想要的角色
+    sid = 7
+    text = f"[JA]{text}[JA]" if is_japanese(text) else f"[ZH]{text}[ZH]"
+    #seq = text_to_sequence(text, symbols=hps.symbols, cleaner_names=hps.data.text_cleaners)
+    seq = text_to_sequence(text, cleaner_names=hps.data.text_cleaners)
+    if hps.data.add_blank:
+        seq = commons.intersperse(seq, 0)
+    with torch.no_grad():
+        x = np.array([seq], dtype=np.int64)
+        x_len = np.array([x.shape[1]], dtype=np.int64)
+        sid = np.array([sid], dtype=np.int64)
+        scales = np.array([0.667, 0.7, 1], dtype=np.float32)
+        scales.resize(1, 3)
+        ort_inputs = {
+                    'input': x,
+                    'input_lengths': x_len,
+                    'scales': scales,
+                    'sid': sid
+                }
+        t1 = time.time()
+        audio = np.squeeze(ort_sess.run(None, ort_inputs))
+        audio *= 32767.0 / max(0.01, np.max(np.abs(audio))) * 0.6
+        audio = np.clip(audio, -32767.0, 32767.0)
+        bytes_wav = bytes()
+        byte_io = io.BytesIO(bytes_wav)
+        wavfile.write(args.audio + '.wav',hps.data.sampling_rate, audio.astype(np.int16))
+        i = 0
+        while i < 19:
+            i +=1
+            cmd = 'ffmpeg -y -i ' +  args.audio + '.wav' + ' -ar 44100 '+ args.audio.replace('temp','temp'+str(i))
+            os.system(cmd)
+        t2 = time.time()
+        print("推理耗时:",(t2 - t1),"s")
+    return text
+tokenizer = AutoTokenizer.from_pretrained(args.ChatGLM, trust_remote_code=True)
+#8G GPU
+model = AutoModel.from_pretrained(args.ChatGLM, trust_remote_code=True).half().quantize(4).cuda()
+history = []
+def send_message():
+    global history
+    message = input_box.get("1.0", "end-1c") # 获取用户输入的文本
+    t1 = time.time()
+    if message == 'clear':
+      history = []
+    else:
+      response, new_history = model.chat(tokenizer, message, history)
+      response = response.replace(" ",'').replace("\n",'.')
+      text = infer(response)
+      text = text.replace('[JA]','').replace('[ZH]','')
+      chat_box.configure(state='normal') # 配置聊天框为可写状态
+      chat_box.insert(tk.END, "You: " + message + "\n") # 在聊天框中显示用户输入的文本
+      chat_box.insert(tk.END, "Tamao: " + text + "\n") # 在聊天框中显示 chatbot 的回复
+    chat_box.configure(state='disabled') # 配置聊天框为只读状态
+    input_box.delete("1.0", tk.END) # 清空输入框
+    t2 = time.time()
+    print("总共耗时:",(t2 - t1),"s")
+
+root = tk.Tk()
+root.title("Tamao")
+
+# 创建聊天框
+chat_box = scrolledtext.ScrolledText(root, width=50, height=10)
+chat_box.configure(state='disabled') # 聊天框一开始是只读状态
+chat_box.pack(side=tk.TOP, fill=tk.BOTH, padx=10, pady=10, expand=True)
+
+# 创建输入框和发送按钮
+input_frame = tk.Frame(root)
+input_frame.pack(side=tk.BOTTOM, fill=tk.X, padx=10, pady=10)
+input_box = tk.Text(input_frame, height=3, width=50) # 设置输入框宽度为50
+input_box.pack(side=tk.LEFT, fill=tk.X, padx=10, expand=True)
+send_button = tk.Button(input_frame, text="Send", command=send_message)
+send_button.pack(side=tk.RIGHT, padx=10)
+
+# 运行主程序
+root.mainloop()

losses.py

@@ -0,0 +1,58 @@
+import torch
+
+
+def feature_loss(fmap_r, fmap_g):
+    loss = 0
+    for dr, dg in zip(fmap_r, fmap_g):
+        for rl, gl in zip(dr, dg):
+            rl = rl.float().detach()
+            gl = gl.float()
+            loss += torch.mean(torch.abs(rl - gl))
+
+    return loss * 2
+
+
+def discriminator_loss(disc_real_outputs, disc_generated_outputs):
+    loss = 0
+    r_losses = []
+    g_losses = []
+    for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
+        dr = dr.float()
+        dg = dg.float()
+        r_loss = torch.mean((1 - dr)**2)
+        g_loss = torch.mean(dg**2)
+        loss += (r_loss + g_loss)
+        r_losses.append(r_loss.item())
+        g_losses.append(g_loss.item())
+
+    return loss, r_losses, g_losses
+
+
+def generator_loss(disc_outputs):
+    loss = 0
+    gen_losses = []
+    for dg in disc_outputs:
+        dg = dg.float()
+        l = torch.mean((1 - dg)**2)
+        gen_losses.append(l)
+        loss += l
+
+    return loss, gen_losses
+
+
+def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
+    """
+  z_p, logs_q: [b, h, t_t]
+  m_p, logs_p: [b, h, t_t]
+  """
+    z_p = z_p.float()
+    logs_q = logs_q.float()
+    m_p = m_p.float()
+    logs_p = logs_p.float()
+    z_mask = z_mask.float()
+
+    kl = logs_p - logs_q - 0.5
+    kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p)
+    kl = torch.sum(kl * z_mask)
+    l = kl / torch.sum(z_mask)
+    return l

main.py

@@ -0,0 +1,251 @@
+import logging
+logging.getLogger('numba').setLevel(logging.WARNING)
+logging.getLogger('matplotlib').setLevel(logging.WARNING)
+logging.getLogger('urllib3').setLevel(logging.WARNING)
+from text import text_to_sequence
+import numpy as np
+from scipy.io import wavfile
+import torch
+import json
+import commons
+import utils
+import sys
+import pathlib
+import onnxruntime as ort
+import gradio as gr
+import argparse
+import time
+import os
+import io
+from scipy.io.wavfile import write
+from flask import Flask, request
+from threading import Thread
+import openai
+import requests
+class VitsGradio:
+    def __init__(self):
+        self.lan = ["中文","日文","自动"]
+        self.chatapi = ["gpt-3.5-turbo","gpt3"]
+        self.modelPaths = []
+        for root,dirs,files in os.walk("checkpoints"):
+            for dir in dirs:
+                self.modelPaths.append(dir)
+        with gr.Blocks() as self.Vits:
+            with gr.Tab("调试用"):
+                with gr.Row():
+                    with gr.Column():
+                        with gr.Row():
+                            with gr.Column():
+                                self.text = gr.TextArea(label="Text", value="你好")
+                                with gr.Accordion(label="测试api", open=False):
+                                    self.local_chat1 = gr.Checkbox(value=False, label="使用网址+文本进行模拟")
+                                    self.url_input = gr.TextArea(label="键入测试", value="http://127.0.0.1:8080/chat?Text=")
+                                    butto = gr.Button("模拟前端抓取语音文件")
+                                btnVC = gr.Button("测试tts+对话程序")
+                            with gr.Column():
+                                output2 = gr.TextArea(label="回复")
+                                output1 = gr.Audio(label="采样率22050")
+                                output3 = gr.outputs.File(label="44100hz: output.wav")
+                butto.click(self.Simul, inputs=[self.text, self.url_input], outputs=[output2,output3])
+                btnVC.click(self.tts_fn, inputs=[self.text], outputs=[output1,output2])
+            with gr.Tab("控制面板"):
+                with gr.Row():
+                    with gr.Column():
+                        with gr.Row():
+                            with gr.Column():
+                                self.api_input1 = gr.TextArea(label="输入api-key或本地存储说话模型的路径", value="https://platform.openai.com/account/api-keys")
+                                with gr.Accordion(label="chatbot选择", open=False):
+                                                self.api_input2 = gr.Checkbox(value=True, label="采用gpt3.5")
+                                                self.local_chat1 = gr.Checkbox(value=False, label="启动本地chatbot")
+                                                self.local_chat2 = gr.Checkbox(value=True, label="是否量化")
+                                                res = gr.TextArea()
+                                Botselection = gr.Button("完成chatbot设定")
+                                Botselection.click(self.check_bot, inputs=[self.api_input1,self.api_input2,self.local_chat1,self.local_chat2], outputs = [res])
+                                self.input1 = gr.Dropdown(label = "模型", choices = self.modelPaths, value = self.modelPaths[0], type = "value")
+                                self.input2 = gr.Dropdown(label="Language", choices=self.lan, value="自动", interactive=True)
+                            with gr.Column():
+                                btnVC = gr.Button("完成vits TTS端设定")
+                                self.input3 = gr.Dropdown(label="Speaker", choices=list(range(101)), value=0, interactive=True)
+                                self.input4 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声比例(noise scale)，以控制情感", value=0.267)
+                                self.input5 = gr.Slider(minimum=0, maximum=1.0, label="更改噪声偏差(noise scale w)，以控制音素长短", value=0.7)
+                                self.input6 = gr.Slider(minimum=0.1, maximum=10, label="duration", value=1)
+                                statusa = gr.TextArea()
+                btnVC.click(self.create_tts_fn, inputs=[self.input1, self.input2, self.input3, self.input4, self.input5, self.input6], outputs = [statusa])
+
+    def Simul(self,text,url_input):
+        web = url_input + text
+        res = requests.get(web)
+        music = res.content
+        with open('output.wav', 'wb') as code:
+            code.write(music)
+        file_path = "output.wav"
+        return web,file_path
+
+
+    def chatgpt(self,text):
+        self.messages.append({"role": "user", "content": text},)
+        chat = openai.ChatCompletion.create(model="gpt-3.5-turbo", messages= self.messages)
+        reply = chat.choices[0].message.content
+        return reply
+    
+    def ChATGLM(self,text):
+        if text == 'clear':
+            self.history = []
+        response, new_history = self.model.chat(self.tokenizer, text, self.history)
+        response = response.replace(" ",'').replace("\n",'.')
+        self.history = new_history
+        return response
+    
+    def gpt3_chat(self,text):
+        call_name = "Waifu"
+        openai.api_key = args.key
+        identity = ""
+        start_sequence = '\n'+str(call_name)+':'
+        restart_sequence = "\nYou: "
+        if 1 == 1:
+            prompt0 = text #当期prompt
+        if text == 'quit':
+            return prompt0
+        prompt = identity + prompt0 + start_sequence
+        response = openai.Completion.create(
+            model="text-davinci-003",
+            prompt=prompt,
+            temperature=0.5,
+            max_tokens=1000,
+            top_p=1.0,
+            frequency_penalty=0.5,
+            presence_penalty=0.0,
+            stop=["\nYou:"]
+        )
+        return response['choices'][0]['text'].strip()
+    
+    def check_bot(self,api_input1,api_input2,local_chat1,local_chat2):
+        if local_chat1:
+            from transformers import AutoTokenizer, AutoModel
+            self.tokenizer = AutoTokenizer.from_pretrained(api_input1, trust_remote_code=True)
+            if local_chat2:
+                self.model = AutoModel.from_pretrained(api_input1, trust_remote_code=True).half().cuda()
+            else:
+                self.model = AutoModel.from_pretrained(api_input1, trust_remote_code=True)
+            self.history = []
+        else:
+            self.messages = []
+            openai.api_key = api_input1
+        return "Finished"
+    
+    def is_japanese(self,string):
+        for ch in string:
+            if ord(ch) > 0x3040 and ord(ch) < 0x30FF:
+                return True
+        return False
+    
+    def is_english(self,string):
+        import re
+        pattern = re.compile('^[A-Za-z0-9.,:;!?()_*"\' ]+$')
+        if pattern.fullmatch(string):
+            return True
+        else:
+            return False
+
+    def get_symbols_from_json(self,path):
+        assert os.path.isfile(path)
+        with open(path, 'r') as f:
+            data = json.load(f)
+        return data['symbols']
+
+    def sle(self,language,text):
+        text = text.replace('\n','。').replace(' ',',')
+        if language == "中文":
+            tts_input1 = "[ZH]" + text + "[ZH]"
+            return tts_input1
+        elif language == "自动":
+            tts_input1 = f"[JA]{text}[JA]" if self.is_japanese(text) else f"[ZH]{text}[ZH]"
+            return tts_input1
+        elif language == "日文":
+            tts_input1 = "[JA]" + text + "[JA]"
+            return tts_input1
+
+    def get_text(self,text,hps_ms):
+        text_norm = text_to_sequence(text,hps_ms.data.text_cleaners)
+        if hps_ms.data.add_blank:
+            text_norm = commons.intersperse(text_norm, 0)
+        text_norm = torch.LongTensor(text_norm)
+        return text_norm
+
+    def create_tts_fn(self,path, input2, input3, n_scale= 0.667,n_scale_w = 0.8, l_scale = 1 ):
+        self.symbols = self.get_symbols_from_json(f"checkpoints/{path}/config.json")
+        self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json")
+        phone_dict = {
+                symbol: i for i, symbol in enumerate(self.symbols)
+            }
+        self.ort_sess = ort.InferenceSession(f"checkpoints/{path}/model.onnx")
+        self.language = input2
+        self.speaker_id = input3
+        self.n_scale = n_scale
+        self.n_scale_w = n_scale_w
+        self.l_scale = l_scale
+        print(self.language,self.speaker_id,self.n_scale)
+        return 'success'
+    
+    def tts_fn(self,text):
+        if self.local_chat1:
+            text = self.chatgpt(text)
+        elif self.api_input2:
+            text = self.ChATGLM(text)
+        else:
+            text = self.gpt3_chat(text)
+        print(text)
+        text =self.sle(self.language,text)
+        seq = text_to_sequence(text, cleaner_names=self.hps.data.text_cleaners)
+        if self.hps.data.add_blank:
+            seq = commons.intersperse(seq, 0)
+        with torch.no_grad():
+            x = np.array([seq], dtype=np.int64)
+            x_len = np.array([x.shape[1]], dtype=np.int64)
+            sid = np.array([self.speaker_id], dtype=np.int64)
+            scales = np.array([self.n_scale, self.n_scale_w, self.l_scale], dtype=np.float32)
+            scales.resize(1, 3)
+            ort_inputs = {
+                        'input': x,
+                        'input_lengths': x_len,
+                        'scales': scales,
+                        'sid': sid
+                    }
+            t1 = time.time()
+            audio = np.squeeze(self.ort_sess.run(None, ort_inputs))
+            audio *= 32767.0 / max(0.01, np.max(np.abs(audio))) * 0.6
+            audio = np.clip(audio, -32767.0, 32767.0)
+            t2 = time.time()
+            spending_time = "推理时间："+str(t2-t1)+"s" 
+            print(spending_time)
+            bytes_wav = bytes()
+            byte_io = io.BytesIO(bytes_wav)
+            wavfile.write('moe/temp1.wav',self.hps.data.sampling_rate, audio.astype(np.int16))
+            cmd = 'ffmpeg -y -i '  + 'moe/temp1.wav' + ' -ar 44100 ' + 'moe/temp2.wav'
+            os.system(cmd)    
+        return (self.hps.data.sampling_rate, audio),text.replace('[JA]','').replace('[ZH]','')
+
+app = Flask(__name__)
+print("开始部署")
+grVits = VitsGradio()
+
+@app.route('/chat')
+def text_api():
+    message = request.args.get('Text','')
+    audio,text = grVits.tts_fn(message)
+    text = text.replace('[JA]','').replace('[ZH]','')
+    with open('moe/temp2.wav','rb') as bit:
+        wav_bytes = bit.read()
+    headers = {
+            'Content-Type': 'audio/wav',
+            'Text': text.encode('utf-8')}
+    return wav_bytes, 200, headers
+
+def gradio_interface():
+    return grVits.Vits.launch()
+
+if __name__ == '__main__':
+    api_thread = Thread(target=app.run, args=("0.0.0.0", 8080))
+    gradio_thread = Thread(target=gradio_interface)
+    api_thread.start()
+    gradio_thread.start()

mel_processing.py

@@ -0,0 +1,137 @@
+import torch
+import torch.nn.functional as F
+import torch.utils.data
+from librosa.filters import mel as librosa_mel_fn
+
+MAX_WAV_VALUE = 32768.0
+
+
+def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
+    """
+    PARAMS
+    ------
+    C: compression factor
+    """
+    return torch.log(torch.clamp(x, min=clip_val) * C)
+
+
+def dynamic_range_decompression_torch(x, C=1):
+    """
+    PARAMS
+    ------
+    C: compression factor used to compress
+    """
+    return torch.exp(x) / C
+
+
+def spectral_normalize_torch(magnitudes):
+    output = dynamic_range_compression_torch(magnitudes)
+    return output
+
+
+def spectral_de_normalize_torch(magnitudes):
+    output = dynamic_range_decompression_torch(magnitudes)
+    return output
+
+
+mel_basis = {}
+hann_window = {}
+
+
+def spectrogram_torch(y,
+                      n_fft,
+                      sampling_rate,
+                      hop_size,
+                      win_size,
+                      center=False):
+    if torch.min(y) < -1.:
+        print('min value is ', torch.min(y))
+    if torch.max(y) > 1.:
+        print('max value is ', torch.max(y))
+
+    global hann_window
+    dtype_device = str(y.dtype) + '_' + str(y.device)
+    wnsize_dtype_device = str(win_size) + '_' + dtype_device
+    if wnsize_dtype_device not in hann_window:
+        hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
+            dtype=y.dtype, device=y.device)
+
+    y = F.pad(y.unsqueeze(1),
+              (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
+              mode='reflect')
+    y = y.squeeze(1)
+
+    spec = torch.stft(y,
+                      n_fft,
+                      hop_length=hop_size,
+                      win_length=win_size,
+                      window=hann_window[wnsize_dtype_device],
+                      center=center,
+                      pad_mode='reflect',
+                      normalized=False,
+                      onesided=True)
+
+    spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
+    return spec
+
+
+def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
+    global mel_basis
+    dtype_device = str(spec.dtype) + '_' + str(spec.device)
+    fmax_dtype_device = str(fmax) + '_' + dtype_device
+    if fmax_dtype_device not in mel_basis:
+        mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
+        mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
+            dtype=spec.dtype, device=spec.device)
+    spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
+    spec = spectral_normalize_torch(spec)
+    return spec
+
+
+def mel_spectrogram_torch(y,
+                          n_fft,
+                          num_mels,
+                          sampling_rate,
+                          hop_size,
+                          win_size,
+                          fmin,
+                          fmax,
+                          center=False):
+    if torch.min(y) < -1.:
+        print('min value is ', torch.min(y))
+    if torch.max(y) > 1.:
+        print('max value is ', torch.max(y))
+
+    global mel_basis, hann_window
+    dtype_device = str(y.dtype) + '_' + str(y.device)
+    fmax_dtype_device = str(fmax) + '_' + dtype_device
+    wnsize_dtype_device = str(win_size) + '_' + dtype_device
+    if fmax_dtype_device not in mel_basis:
+        mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
+        mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
+            dtype=y.dtype, device=y.device)
+    if wnsize_dtype_device not in hann_window:
+        hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
+            dtype=y.dtype, device=y.device)
+
+    y = F.pad(y.unsqueeze(1),
+              (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
+              mode='reflect')
+    y = y.squeeze(1)
+
+    spec = torch.stft(y,
+                      n_fft,
+                      hop_length=hop_size,
+                      win_length=win_size,
+                      window=hann_window[wnsize_dtype_device],
+                      center=center,
+                      pad_mode='reflect',
+                      normalized=False,
+                      onesided=True)
+
+    spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
+
+    spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
+    spec = spectral_normalize_torch(spec)
+
+    return spec

models.py

@@ -2,18 +2,25 @@ import math
 
 import torch
 from torch import nn
-from torch.nn import Conv1d, ConvTranspose1d, Conv2d
 from torch.nn import functional as F
+from torch.nn import Conv1d, ConvTranspose1d, Conv2d
 from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
+import monotonic_align
 
-import attentions
 import commons
 import modules
+import attentions
 from commons import init_weights, get_padding
 
 
 class StochasticDurationPredictor(nn.Module):
-    def __init__(self, in_channels, filter_channels, kernel_size, p_dropout, n_flows=4, gin_channels=0):
+    def __init__(self,
+                 in_channels,
+                 filter_channels,
+                 kernel_size,
+                 p_dropout,
+                 n_flows=4,
+                 gin_channels=0):
         super().__init__()
         filter_channels = in_channels  # it needs to be removed from future version.
         self.in_channels = in_channels
@@ -27,25 +34,39 @@ class StochasticDurationPredictor(nn.Module):
         self.flows = nn.ModuleList()
         self.flows.append(modules.ElementwiseAffine(2))
         for i in range(n_flows):
-            self.flows.append(modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
+            self.flows.append(
+                modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
             self.flows.append(modules.Flip())
 
         self.post_pre = nn.Conv1d(1, filter_channels, 1)
         self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1)
-        self.post_convs = modules.DDSConv(filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout)
+        self.post_convs = modules.DDSConv(filter_channels,
+                                          kernel_size,
+                                          n_layers=3,
+                                          p_dropout=p_dropout)
         self.post_flows = nn.ModuleList()
         self.post_flows.append(modules.ElementwiseAffine(2))
         for i in range(4):
-            self.post_flows.append(modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
+            self.post_flows.append(
+                modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
             self.post_flows.append(modules.Flip())
 
         self.pre = nn.Conv1d(in_channels, filter_channels, 1)
         self.proj = nn.Conv1d(filter_channels, filter_channels, 1)
-        self.convs = modules.DDSConv(filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout)
+        self.convs = modules.DDSConv(filter_channels,
+                                     kernel_size,
+                                     n_layers=3,
+                                     p_dropout=p_dropout)
         if gin_channels != 0:
             self.cond = nn.Conv1d(gin_channels, filter_channels, 1)
 
-    def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0):
+    def forward(self,
+                x,
+                x_mask,
+                w=None,
+                g=None,
+                reverse=False,
+                noise_scale=1.0):
         x = torch.detach(x)
         x = self.pre(x)
         if g is not None:
@@ -62,7 +83,8 @@ class StochasticDurationPredictor(nn.Module):
             h_w = self.post_pre(w)
             h_w = self.post_convs(h_w, x_mask)
             h_w = self.post_proj(h_w) * x_mask
-            e_q = torch.randn(w.size(0), 2, w.size(2)).to(device=x.device, dtype=x.dtype) * x_mask
+            e_q = torch.randn(w.size(0), 2, w.size(2)).to(
+                device=x.device, dtype=x.dtype) * x_mask
             z_q = e_q
             for flow in self.post_flows:
                 z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))
@@ -70,8 +92,11 @@ class StochasticDurationPredictor(nn.Module):
             z_u, z1 = torch.split(z_q, [1, 1], 1)
             u = torch.sigmoid(z_u) * x_mask
             z0 = (w - u) * x_mask
-            logdet_tot_q += torch.sum((F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1, 2])
-            logq = torch.sum(-0.5 * (math.log(2 * math.pi) + (e_q ** 2)) * x_mask, [1, 2]) - logdet_tot_q
+            logdet_tot_q += torch.sum(
+                (F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1, 2])
+            logq = torch.sum(
+                -0.5 * (math.log(2 * math.pi) +
+                        (e_q**2)) * x_mask, [1, 2]) - logdet_tot_q
 
             logdet_tot = 0
             z0, logdet = self.log_flow(z0, x_mask)
@@ -80,12 +105,14 @@ class StochasticDurationPredictor(nn.Module):
             for flow in flows:
                 z, logdet = flow(z, x_mask, g=x, reverse=reverse)
                 logdet_tot = logdet_tot + logdet
-            nll = torch.sum(0.5 * (math.log(2 * math.pi) + (z ** 2)) * x_mask, [1, 2]) - logdet_tot
+            nll = torch.sum(0.5 * (math.log(2 * math.pi) +
+                                   (z**2)) * x_mask, [1, 2]) - logdet_tot
             return nll + logq  # [b]
         else:
             flows = list(reversed(self.flows))
             flows = flows[:-2] + [flows[-1]]  # remove a useless vflow
-            z = torch.randn(x.size(0), 2, x.size(2)).to(device=x.device, dtype=x.dtype) * noise_scale
+            z = torch.randn(x.size(0), 2, x.size(2)).to(
+                device=x.device, dtype=x.dtype) * noise_scale
             for flow in flows:
                 z = flow(z, x_mask, g=x, reverse=reverse)
             z0, z1 = torch.split(z, [1, 1], 1)
@@ -94,7 +121,12 @@ class StochasticDurationPredictor(nn.Module):
 
 
 class DurationPredictor(nn.Module):
-    def __init__(self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0):
+    def __init__(self,
+                 in_channels,
+                 filter_channels,
+                 kernel_size,
+                 p_dropout,
+                 gin_channels=0):
         super().__init__()
 
         self.in_channels = in_channels
@@ -104,9 +136,15 @@ class DurationPredictor(nn.Module):
         self.gin_channels = gin_channels
 
         self.drop = nn.Dropout(p_dropout)
-        self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size // 2)
+        self.conv_1 = nn.Conv1d(in_channels,
+                                filter_channels,
+                                kernel_size,
+                                padding=kernel_size // 2)
         self.norm_1 = modules.LayerNorm(filter_channels)
-        self.conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size // 2)
+        self.conv_2 = nn.Conv1d(filter_channels,
+                                filter_channels,
+                                kernel_size,
+                                padding=kernel_size // 2)
         self.norm_2 = modules.LayerNorm(filter_channels)
         self.proj = nn.Conv1d(filter_channels, 1, 1)
 
@@ -131,15 +169,8 @@ class DurationPredictor(nn.Module):
 
 
 class TextEncoder(nn.Module):
-    def __init__(self,
-                 n_vocab,
-                 out_channels,
-                 hidden_channels,
-                 filter_channels,
-                 n_heads,
-                 n_layers,
-                 kernel_size,
-                 p_dropout):
+    def __init__(self, n_vocab, out_channels, hidden_channels, filter_channels,
+                 n_heads, n_layers, kernel_size, p_dropout):
         super().__init__()
         self.n_vocab = n_vocab
         self.out_channels = out_channels
@@ -150,24 +181,19 @@ class TextEncoder(nn.Module):
         self.kernel_size = kernel_size
         self.p_dropout = p_dropout
 
-        if self.n_vocab != 0:
-            self.emb = nn.Embedding(n_vocab, hidden_channels)
-            nn.init.normal_(self.emb.weight, 0.0, hidden_channels ** -0.5)
-
-        self.encoder = attentions.Encoder(
-            hidden_channels,
-            filter_channels,
-            n_heads,
-            n_layers,
-            kernel_size,
-            p_dropout)
+        self.emb = nn.Embedding(n_vocab, hidden_channels)
+        nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5)
+
+        self.encoder = attentions.Encoder(hidden_channels, filter_channels,
+                                          n_heads, n_layers, kernel_size,
+                                          p_dropout)
         self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
 
     def forward(self, x, x_lengths):
-        if self.n_vocab != 0:
-            x = self.emb(x) * math.sqrt(self.hidden_channels)  # [b, t, h]
+        x = self.emb(x) * math.sqrt(self.hidden_channels)  # [b, t, h]
         x = torch.transpose(x, 1, -1)  # [b, h, t]
-        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
+        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)),
+                                 1).to(x.dtype)
 
         x = self.encoder(x * x_mask, x_mask)
         stats = self.proj(x) * x_mask
@@ -197,8 +223,13 @@ class ResidualCouplingBlock(nn.Module):
         self.flows = nn.ModuleList()
         for i in range(n_flows):
             self.flows.append(
-                modules.ResidualCouplingLayer(channels, hidden_channels, kernel_size, dilation_rate, n_layers,
-                                              gin_channels=gin_channels, mean_only=True))
+                modules.ResidualCouplingLayer(channels,
+                                              hidden_channels,
+                                              kernel_size,
+                                              dilation_rate,
+                                              n_layers,
+                                              gin_channels=gin_channels,
+                                              mean_only=True))
             self.flows.append(modules.Flip())
 
     def forward(self, x, x_mask, g=None, reverse=False):
@@ -230,11 +261,16 @@ class PosteriorEncoder(nn.Module):
         self.gin_channels = gin_channels
 
         self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
-        self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels)
+        self.enc = modules.WN(hidden_channels,
+                              kernel_size,
+                              dilation_rate,
+                              n_layers,
+                              gin_channels=gin_channels)
         self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
 
     def forward(self, x, x_lengths, g=None):
-        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
+        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)),
+                                 1).to(x.dtype)
         x = self.pre(x) * x_mask
         x = self.enc(x, x_mask, g=g)
         stats = self.proj(x) * x_mask
@@ -244,24 +280,40 @@ class PosteriorEncoder(nn.Module):
 
 
 class Generator(torch.nn.Module):
-    def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates,
-                 upsample_initial_channel, upsample_kernel_sizes, gin_channels=0):
+    def __init__(self,
+                 initial_channel,
+                 resblock,
+                 resblock_kernel_sizes,
+                 resblock_dilation_sizes,
+                 upsample_rates,
+                 upsample_initial_channel,
+                 upsample_kernel_sizes,
+                 gin_channels=0):
         super(Generator, self).__init__()
         self.num_kernels = len(resblock_kernel_sizes)
         self.num_upsamples = len(upsample_rates)
-        self.conv_pre = Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3)
+        self.conv_pre = Conv1d(initial_channel,
+                               upsample_initial_channel,
+                               7,
+                               1,
+                               padding=3)
         resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2
 
         self.ups = nn.ModuleList()
         for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
-            self.ups.append(weight_norm(
-                ConvTranspose1d(upsample_initial_channel // (2 ** i), upsample_initial_channel // (2 ** (i + 1)),
-                                k, u, padding=(k - u) // 2)))
+            self.ups.append(
+                weight_norm(
+                    ConvTranspose1d(upsample_initial_channel // (2**i),
+                                    upsample_initial_channel // (2**(i + 1)),
+                                    k,
+                                    u,
+                                    padding=(k - u) // 2)))
 
         self.resblocks = nn.ModuleList()
         for i in range(len(self.ups)):
-            ch = upsample_initial_channel // (2 ** (i + 1))
-            for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
+            ch = upsample_initial_channel // (2**(i + 1))
+            for j, (k, d) in enumerate(
+                    zip(resblock_kernel_sizes, resblock_dilation_sizes)):
                 self.resblocks.append(resblock(ch, k, d))
 
         self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
@@ -300,17 +352,37 @@ class Generator(torch.nn.Module):
 
 
 class DiscriminatorP(torch.nn.Module):
-    def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+    def __init__(self,
+                 period,
+                 kernel_size=5,
+                 stride=3,
+                 use_spectral_norm=False):
         super(DiscriminatorP, self).__init__()
         self.period = period
         self.use_spectral_norm = use_spectral_norm
-        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        norm_f = weight_norm if use_spectral_norm is False else spectral_norm
         self.convs = nn.ModuleList([
-            norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
-            norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
-            norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
-            norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
-            norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(
+                Conv2d(1,
+                       32, (kernel_size, 1), (stride, 1),
+                       padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(
+                Conv2d(32,
+                       128, (kernel_size, 1), (stride, 1),
+                       padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(
+                Conv2d(128,
+                       512, (kernel_size, 1), (stride, 1),
+                       padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(
+                Conv2d(512,
+                       1024, (kernel_size, 1), (stride, 1),
+                       padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(
+                Conv2d(1024,
+                       1024, (kernel_size, 1),
+                       1,
+                       padding=(get_padding(kernel_size, 1), 0))),
         ])
         self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
 
@@ -339,7 +411,7 @@ class DiscriminatorP(torch.nn.Module):
 class DiscriminatorS(torch.nn.Module):
     def __init__(self, use_spectral_norm=False):
         super(DiscriminatorS, self).__init__()
-        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        norm_f = weight_norm if use_spectral_norm is False else spectral_norm
         self.convs = nn.ModuleList([
             norm_f(Conv1d(1, 16, 15, 1, padding=7)),
             norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
@@ -370,7 +442,10 @@ class MultiPeriodDiscriminator(torch.nn.Module):
         periods = [2, 3, 5, 7, 11]
 
         discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
-        discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods]
+        discs = discs + [
+            DiscriminatorP(i, use_spectral_norm=use_spectral_norm)
+            for i in periods
+        ]
         self.discriminators = nn.ModuleList(discs)
 
     def forward(self, y, y_hat):
@@ -391,9 +466,8 @@ class MultiPeriodDiscriminator(torch.nn.Module):
 
 class SynthesizerTrn(nn.Module):
     """
-    Synthesizer for Training
-    """
-
+  Synthesizer for Training
+  """
     def __init__(self,
                  n_vocab,
                  spec_channels,
@@ -435,32 +509,116 @@ class SynthesizerTrn(nn.Module):
         self.segment_size = segment_size
         self.n_speakers = n_speakers
         self.gin_channels = gin_channels
+        if self.n_speakers != 0:
+            message = "gin_channels must be none zero for multiple speakers"
+            assert gin_channels != 0, message
 
         self.use_sdp = use_sdp
 
-        self.enc_p = TextEncoder(n_vocab,
-                                 inter_channels,
-                                 hidden_channels,
-                                 filter_channels,
-                                 n_heads,
-                                 n_layers,
-                                 kernel_size,
-                                 p_dropout)
-        self.dec = Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates,
-                             upsample_initial_channel, upsample_kernel_sizes, gin_channels=gin_channels)
-        self.enc_q = PosteriorEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16,
+        self.enc_p = TextEncoder(n_vocab, inter_channels, hidden_channels,
+                                 filter_channels, n_heads, n_layers,
+                                 kernel_size, p_dropout)
+        self.dec = Generator(inter_channels,
+                             resblock,
+                             resblock_kernel_sizes,
+                             resblock_dilation_sizes,
+                             upsample_rates,
+                             upsample_initial_channel,
+                             upsample_kernel_sizes,
+                             gin_channels=gin_channels)
+        self.enc_q = PosteriorEncoder(spec_channels,
+                                      inter_channels,
+                                      hidden_channels,
+                                      5,
+                                      1,
+                                      16,
                                       gin_channels=gin_channels)
-        self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4, gin_channels=gin_channels)
+        self.flow = ResidualCouplingBlock(inter_channels,
+                                          hidden_channels,
+                                          5,
+                                          1,
+                                          4,
+                                          gin_channels=gin_channels)
 
         if use_sdp:
-            self.dp = StochasticDurationPredictor(hidden_channels, 192, 3, 0.5, 4, gin_channels=gin_channels)
+            self.dp = StochasticDurationPredictor(hidden_channels,
+                                                  192,
+                                                  3,
+                                                  0.5,
+                                                  4,
+                                                  gin_channels=gin_channels)
         else:
-            self.dp = DurationPredictor(hidden_channels, 256, 3, 0.5, gin_channels=gin_channels)
+            self.dp = DurationPredictor(hidden_channels,
+                                        256,
+                                        3,
+                                        0.5,
+                                        gin_channels=gin_channels)
 
         if n_speakers > 1:
             self.emb_g = nn.Embedding(n_speakers, gin_channels)
 
-    def infer(self, x, x_lengths, sid=None, noise_scale=1, length_scale=1, noise_scale_w=1., max_len=None):
+    def forward(self, x, x_lengths, y, y_lengths, sid=None):
+
+        x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths)
+        if self.n_speakers > 0:
+            g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
+        else:
+            g = None
+
+        z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
+        z_p = self.flow(z, y_mask, g=g)
+
+        with torch.no_grad():
+            # negative cross-entropy
+            s_p_sq_r = torch.exp(-2 * logs_p)  # [b, d, t]
+            neg_cent1 = torch.sum(-0.5 * math.log(2 * math.pi) - logs_p, [1],
+                                  keepdim=True)  # [b, 1, t_s]
+            neg_cent2 = torch.matmul(
+                -0.5 * (z_p**2).transpose(1, 2),
+                s_p_sq_r)  # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
+            neg_cent3 = torch.matmul(
+                z_p.transpose(1, 2),
+                (m_p * s_p_sq_r))  # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
+            neg_cent4 = torch.sum(-0.5 * (m_p**2) * s_p_sq_r, [1],
+                                  keepdim=True)  # [b, 1, t_s]
+            neg_cent = neg_cent1 + neg_cent2 + neg_cent3 + neg_cent4
+
+            attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(
+                y_mask, -1)
+            attn = monotonic_align.maximum_path(
+                neg_cent, attn_mask.squeeze(1)).unsqueeze(1).detach()
+
+        w = attn.sum(2)
+        if self.use_sdp:
+            l_length = self.dp(x, x_mask, w, g=g)
+            l_length = l_length / torch.sum(x_mask)
+        else:
+            logw_ = torch.log(w + 1e-6) * x_mask
+            logw = self.dp(x, x_mask, g=g)
+            l_length = torch.sum(
+                (logw - logw_)**2, [1, 2]) / torch.sum(x_mask)  # for averaging
+
+        # expand prior
+        m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1,
+                                                          2)).transpose(1, 2)
+        logs_p = torch.matmul(attn.squeeze(1),
+                              logs_p.transpose(1, 2)).transpose(1, 2)
+
+        z_slice, ids_slice = commons.rand_slice_segments(
+            z, y_lengths, self.segment_size)
+        o = self.dec(z_slice, g=g)
+        return o, l_length, attn, ids_slice, x_mask, y_mask, (z, z_p, m_p,
+                                                              logs_p, m_q,
+                                                              logs_q)
+
+    def infer(self,
+              x,
+              x_lengths,
+              sid=None,
+              noise_scale=1,
+              length_scale=1,
+              noise_scale_w=1.,
+              max_len=None):
         x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths)
         if self.n_speakers > 0:
             g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
@@ -468,25 +626,41 @@ class SynthesizerTrn(nn.Module):
             g = None
 
         if self.use_sdp:
-            logw = self.dp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w)
+            logw = self.dp(x,
+                           x_mask,
+                           g=g,
+                           reverse=True,
+                           noise_scale=noise_scale_w)
         else:
             logw = self.dp(x, x_mask, g=g)
         w = torch.exp(logw) * x_mask * length_scale
         w_ceil = torch.ceil(w)
         y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
-        y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None), 1).to(x_mask.dtype)
+        y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None),
+                                 1).to(x_mask.dtype)
         attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
         attn = commons.generate_path(w_ceil, attn_mask)
 
-        m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(1, 2)  # [b, t', t], [b, t, d] -> [b, d, t']
-        logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(1,
-                                                                                 2)  # [b, t', t], [b, t, d] -> [b, d, t']
+        m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(
+            1, 2)  # [b, t', t], [b, t, d] -> [b, d, t']
+        logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(
+            1, 2)).transpose(1, 2)  # [b, t', t], [b, t, d] -> [b, d, t']
 
         z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
         z = self.flow(z_p, y_mask, g=g, reverse=True)
         o = self.dec((z * y_mask)[:, :, :max_len], g=g)
         return o, attn, y_mask, (z, z_p, m_p, logs_p)
 
+    def export_forward(self, x, x_lengths, scales, sid):
+        # shape of scales: Bx3, make triton happy
+        audio, *_ = self.infer(x,
+                               x_lengths,
+                               sid,
+                               noise_scale=scales[0][0],
+                               length_scale=scales[0][1],
+                               noise_scale_w=scales[0][2])
+        return audio
+
     def voice_conversion(self, y, y_lengths, sid_src, sid_tgt):
         assert self.n_speakers > 0, "n_speakers have to be larger than 0."
         g_src = self.emb_g(sid_src).unsqueeze(-1)

modules.py

@@ -1,8 +1,8 @@
 import math
+
 import torch
 from torch import nn
 from torch.nn import functional as F
-
 from torch.nn import Conv1d
 from torch.nn.utils import weight_norm, remove_weight_norm
 
@@ -10,197 +10,249 @@ import commons
 from commons import init_weights, get_padding
 from transforms import piecewise_rational_quadratic_transform
 
-
 LRELU_SLOPE = 0.1
 
 
 class LayerNorm(nn.Module):
-  def __init__(self, channels, eps=1e-5):
-    super().__init__()
-    self.channels = channels
-    self.eps = eps
+    def __init__(self, channels, eps=1e-5):
+        super().__init__()
+        self.channels = channels
+        self.eps = eps
 
-    self.gamma = nn.Parameter(torch.ones(channels))
-    self.beta = nn.Parameter(torch.zeros(channels))
+        self.gamma = nn.Parameter(torch.ones(channels))
+        self.beta = nn.Parameter(torch.zeros(channels))
+
+    def forward(self, x):
+        x = x.transpose(1, -1)
+        x = F.layer_norm(x, (self.channels, ), self.gamma, self.beta, self.eps)
+        return x.transpose(1, -1)
 
-  def forward(self, x):
-    x = x.transpose(1, -1)
-    x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
-    return x.transpose(1, -1)
 
- 
 class ConvReluNorm(nn.Module):
-  def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout):
-    super().__init__()
-    self.in_channels = in_channels
-    self.hidden_channels = hidden_channels
-    self.out_channels = out_channels
-    self.kernel_size = kernel_size
-    self.n_layers = n_layers
-    self.p_dropout = p_dropout
-    assert n_layers > 1, "Number of layers should be larger than 0."
-
-    self.conv_layers = nn.ModuleList()
-    self.norm_layers = nn.ModuleList()
-    self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size//2))
-    self.norm_layers.append(LayerNorm(hidden_channels))
-    self.relu_drop = nn.Sequential(
-        nn.ReLU(),
-        nn.Dropout(p_dropout))
-    for _ in range(n_layers-1):
-      self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size//2))
-      self.norm_layers.append(LayerNorm(hidden_channels))
-    self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
-    self.proj.weight.data.zero_()
-    self.proj.bias.data.zero_()
-
-  def forward(self, x, x_mask):
-    x_org = x
-    for i in range(self.n_layers):
-      x = self.conv_layers[i](x * x_mask)
-      x = self.norm_layers[i](x)
-      x = self.relu_drop(x)
-    x = x_org + self.proj(x)
-    return x * x_mask
+    def __init__(self, in_channels, hidden_channels, out_channels, kernel_size,
+                 n_layers, p_dropout):
+        super().__init__()
+        self.in_channels = in_channels
+        self.hidden_channels = hidden_channels
+        self.out_channels = out_channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.p_dropout = p_dropout
+        assert n_layers > 1, "Number of layers should be larger than 0."
+
+        self.conv_layers = nn.ModuleList()
+        self.norm_layers = nn.ModuleList()
+        self.conv_layers.append(
+            nn.Conv1d(in_channels,
+                      hidden_channels,
+                      kernel_size,
+                      padding=kernel_size // 2))
+        self.norm_layers.append(LayerNorm(hidden_channels))
+        self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout))
+        for _ in range(n_layers - 1):
+            self.conv_layers.append(
+                nn.Conv1d(hidden_channels,
+                          hidden_channels,
+                          kernel_size,
+                          padding=kernel_size // 2))
+            self.norm_layers.append(LayerNorm(hidden_channels))
+        self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
+        self.proj.weight.data.zero_()
+        self.proj.bias.data.zero_()
+
+    def forward(self, x, x_mask):
+        x_org = x
+        for i in range(self.n_layers):
+            x = self.conv_layers[i](x * x_mask)
+            x = self.norm_layers[i](x)
+            x = self.relu_drop(x)
+        x = x_org + self.proj(x)
+        return x * x_mask
 
 
 class DDSConv(nn.Module):
-  """
+    """
   Dialted and Depth-Separable Convolution
   """
-  def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
-    super().__init__()
-    self.channels = channels
-    self.kernel_size = kernel_size
-    self.n_layers = n_layers
-    self.p_dropout = p_dropout
-
-    self.drop = nn.Dropout(p_dropout)
-    self.convs_sep = nn.ModuleList()
-    self.convs_1x1 = nn.ModuleList()
-    self.norms_1 = nn.ModuleList()
-    self.norms_2 = nn.ModuleList()
-    for i in range(n_layers):
-      dilation = kernel_size ** i
-      padding = (kernel_size * dilation - dilation) // 2
-      self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size, 
-          groups=channels, dilation=dilation, padding=padding
-      ))
-      self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
-      self.norms_1.append(LayerNorm(channels))
-      self.norms_2.append(LayerNorm(channels))
-
-  def forward(self, x, x_mask, g=None):
-    if g is not None:
-      x = x + g
-    for i in range(self.n_layers):
-      y = self.convs_sep[i](x * x_mask)
-      y = self.norms_1[i](y)
-      y = F.gelu(y)
-      y = self.convs_1x1[i](y)
-      y = self.norms_2[i](y)
-      y = F.gelu(y)
-      y = self.drop(y)
-      x = x + y
-    return x * x_mask
+    def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
+        super().__init__()
+        self.channels = channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.p_dropout = p_dropout
+
+        self.drop = nn.Dropout(p_dropout)
+        self.convs_sep = nn.ModuleList()
+        self.convs_1x1 = nn.ModuleList()
+        self.norms_1 = nn.ModuleList()
+        self.norms_2 = nn.ModuleList()
+        for i in range(n_layers):
+            dilation = kernel_size**i
+            padding = (kernel_size * dilation - dilation) // 2
+            self.convs_sep.append(
+                nn.Conv1d(channels,
+                          channels,
+                          kernel_size,
+                          groups=channels,
+                          dilation=dilation,
+                          padding=padding))
+            self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
+            self.norms_1.append(LayerNorm(channels))
+            self.norms_2.append(LayerNorm(channels))
+
+    def forward(self, x, x_mask, g=None):
+        if g is not None:
+            x = x + g
+        for i in range(self.n_layers):
+            y = self.convs_sep[i](x * x_mask)
+            y = self.norms_1[i](y)
+            y = F.gelu(y)
+            y = self.convs_1x1[i](y)
+            y = self.norms_2[i](y)
+            y = F.gelu(y)
+            y = self.drop(y)
+            x = x + y
+        return x * x_mask
 
 
 class WN(torch.nn.Module):
-  def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0):
-    super(WN, self).__init__()
-    assert(kernel_size % 2 == 1)
-    self.hidden_channels =hidden_channels
-    self.kernel_size = kernel_size,
-    self.dilation_rate = dilation_rate
-    self.n_layers = n_layers
-    self.gin_channels = gin_channels
-    self.p_dropout = p_dropout
-
-    self.in_layers = torch.nn.ModuleList()
-    self.res_skip_layers = torch.nn.ModuleList()
-    self.drop = nn.Dropout(p_dropout)
-
-    if gin_channels != 0:
-      cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1)
-      self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')
-
-    for i in range(n_layers):
-      dilation = dilation_rate ** i
-      padding = int((kernel_size * dilation - dilation) / 2)
-      in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size,
-                                 dilation=dilation, padding=padding)
-      in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
-      self.in_layers.append(in_layer)
-
-      # last one is not necessary
-      if i < n_layers - 1:
-        res_skip_channels = 2 * hidden_channels
-      else:
-        res_skip_channels = hidden_channels
-
-      res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
-      res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
-      self.res_skip_layers.append(res_skip_layer)
-
-  def forward(self, x, x_mask, g=None, **kwargs):
-    output = torch.zeros_like(x)
-    n_channels_tensor = torch.IntTensor([self.hidden_channels])
-
-    if g is not None:
-      g = self.cond_layer(g)
-
-    for i in range(self.n_layers):
-      x_in = self.in_layers[i](x)
-      if g is not None:
-        cond_offset = i * 2 * self.hidden_channels
-        g_l = g[:,cond_offset:cond_offset+2*self.hidden_channels,:]
-      else:
-        g_l = torch.zeros_like(x_in)
-
-      acts = commons.fused_add_tanh_sigmoid_multiply(
-          x_in,
-          g_l,
-          n_channels_tensor)
-      acts = self.drop(acts)
-
-      res_skip_acts = self.res_skip_layers[i](acts)
-      if i < self.n_layers - 1:
-        res_acts = res_skip_acts[:,:self.hidden_channels,:]
-        x = (x + res_acts) * x_mask
-        output = output + res_skip_acts[:,self.hidden_channels:,:]
-      else:
-        output = output + res_skip_acts
-    return output * x_mask
-
-  def remove_weight_norm(self):
-    if self.gin_channels != 0:
-      torch.nn.utils.remove_weight_norm(self.cond_layer)
-    for l in self.in_layers:
-      torch.nn.utils.remove_weight_norm(l)
-    for l in self.res_skip_layers:
-     torch.nn.utils.remove_weight_norm(l)
+    def __init__(self,
+                 hidden_channels,
+                 kernel_size,
+                 dilation_rate,
+                 n_layers,
+                 gin_channels=0,
+                 p_dropout=0):
+        super(WN, self).__init__()
+        assert (kernel_size % 2 == 1)
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size,
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.gin_channels = gin_channels
+        self.p_dropout = p_dropout
+
+        self.in_layers = torch.nn.ModuleList()
+        self.res_skip_layers = torch.nn.ModuleList()
+        self.drop = nn.Dropout(p_dropout)
+
+        if gin_channels != 0:
+            cond_layer = torch.nn.Conv1d(gin_channels,
+                                         2 * hidden_channels * n_layers, 1)
+            self.cond_layer = torch.nn.utils.weight_norm(cond_layer,
+                                                         name='weight')
+
+        for i in range(n_layers):
+            dilation = dilation_rate**i
+            padding = int((kernel_size * dilation - dilation) / 2)
+            in_layer = torch.nn.Conv1d(hidden_channels,
+                                       2 * hidden_channels,
+                                       kernel_size,
+                                       dilation=dilation,
+                                       padding=padding)
+            in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
+            self.in_layers.append(in_layer)
+
+            # last one is not necessary
+            if i < n_layers - 1:
+                res_skip_channels = 2 * hidden_channels
+            else:
+                res_skip_channels = hidden_channels
+
+            res_skip_layer = torch.nn.Conv1d(hidden_channels,
+                                             res_skip_channels, 1)
+            res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer,
+                                                        name='weight')
+            self.res_skip_layers.append(res_skip_layer)
+
+    def forward(self, x, x_mask, g=None, **kwargs):
+        output = torch.zeros_like(x)
+        n_channels_tensor = torch.IntTensor([self.hidden_channels])
+
+        if g is not None:
+            g = self.cond_layer(g)
+
+        for i in range(self.n_layers):
+            x_in = self.in_layers[i](x)
+            if g is not None:
+                cond_offset = i * 2 * self.hidden_channels
+                g_l = g[:,
+                        cond_offset:cond_offset + 2 * self.hidden_channels, :]
+            else:
+                g_l = torch.zeros_like(x_in)
+
+            acts = commons.fused_add_tanh_sigmoid_multiply(
+                x_in, g_l, n_channels_tensor)
+            acts = self.drop(acts)
+
+            res_skip_acts = self.res_skip_layers[i](acts)
+            if i < self.n_layers - 1:
+                res_acts = res_skip_acts[:, :self.hidden_channels, :]
+                x = (x + res_acts) * x_mask
+                output = output + res_skip_acts[:, self.hidden_channels:, :]
+            else:
+                output = output + res_skip_acts
+        return output * x_mask
+
+    def remove_weight_norm(self):
+        if self.gin_channels != 0:
+            torch.nn.utils.remove_weight_norm(self.cond_layer)
+        for l in self.in_layers:
+            torch.nn.utils.remove_weight_norm(l)
+        for l in self.res_skip_layers:
+            torch.nn.utils.remove_weight_norm(l)
 
 
 class ResBlock1(torch.nn.Module):
     def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
         super(ResBlock1, self).__init__()
         self.convs1 = nn.ModuleList([
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
-                               padding=get_padding(kernel_size, dilation[0]))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
-                               padding=get_padding(kernel_size, dilation[1]))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
-                               padding=get_padding(kernel_size, dilation[2])))
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[0],
+                       padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[1],
+                       padding=get_padding(kernel_size, dilation[1]))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[2],
+                       padding=get_padding(kernel_size, dilation[2])))
         ])
         self.convs1.apply(init_weights)
 
         self.convs2 = nn.ModuleList([
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
-                               padding=get_padding(kernel_size, 1))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
-                               padding=get_padding(kernel_size, 1))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
-                               padding=get_padding(kernel_size, 1)))
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=1,
+                       padding=get_padding(kernel_size, 1))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=1,
+                       padding=get_padding(kernel_size, 1))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=1,
+                       padding=get_padding(kernel_size, 1)))
         ])
         self.convs2.apply(init_weights)
 
@@ -230,10 +282,20 @@ class ResBlock2(torch.nn.Module):
     def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
         super(ResBlock2, self).__init__()
         self.convs = nn.ModuleList([
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
-                               padding=get_padding(kernel_size, dilation[0]))),
-            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
-                               padding=get_padding(kernel_size, dilation[1])))
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[0],
+                       padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[1],
+                       padding=get_padding(kernel_size, dilation[1])))
         ])
         self.convs.apply(init_weights)
 
@@ -254,134 +316,154 @@ class ResBlock2(torch.nn.Module):
 
 
 class Log(nn.Module):
-  def forward(self, x, x_mask, reverse=False, **kwargs):
-    if not reverse:
-      y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
-      logdet = torch.sum(-y, [1, 2])
-      return y, logdet
-    else:
-      x = torch.exp(x) * x_mask
-      return x
-    
+    def forward(self, x, x_mask, reverse=False, **kwargs):
+        if not reverse:
+            y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
+            logdet = torch.sum(-y, [1, 2])
+            return y, logdet
+        else:
+            x = torch.exp(x) * x_mask
+            return x
+
 
 class Flip(nn.Module):
-  def forward(self, x, *args, reverse=False, **kwargs):
-    x = torch.flip(x, [1])
-    if not reverse:
-      logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
-      return x, logdet
-    else:
-      return x
+    def forward(self, x, *args, reverse=False, **kwargs):
+        x = torch.flip(x, [1])
+        if not reverse:
+            logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
+            return x, logdet
+        else:
+            return x
 
 
 class ElementwiseAffine(nn.Module):
-  def __init__(self, channels):
-    super().__init__()
-    self.channels = channels
-    self.m = nn.Parameter(torch.zeros(channels,1))
-    self.logs = nn.Parameter(torch.zeros(channels,1))
-
-  def forward(self, x, x_mask, reverse=False, **kwargs):
-    if not reverse:
-      y = self.m + torch.exp(self.logs) * x
-      y = y * x_mask
-      logdet = torch.sum(self.logs * x_mask, [1,2])
-      return y, logdet
-    else:
-      x = (x - self.m) * torch.exp(-self.logs) * x_mask
-      return x
+    def __init__(self, channels):
+        super().__init__()
+        self.channels = channels
+        self.m = nn.Parameter(torch.zeros(channels, 1))
+        self.logs = nn.Parameter(torch.zeros(channels, 1))
+
+    def forward(self, x, x_mask, reverse=False, **kwargs):
+        if not reverse:
+            y = self.m + torch.exp(self.logs) * x
+            y = y * x_mask
+            logdet = torch.sum(self.logs * x_mask, [1, 2])
+            return y, logdet
+        else:
+            x = (x - self.m) * torch.exp(-self.logs) * x_mask
+            return x
 
 
 class ResidualCouplingLayer(nn.Module):
-  def __init__(self,
-      channels,
-      hidden_channels,
-      kernel_size,
-      dilation_rate,
-      n_layers,
-      p_dropout=0,
-      gin_channels=0,
-      mean_only=False):
-    assert channels % 2 == 0, "channels should be divisible by 2"
-    super().__init__()
-    self.channels = channels
-    self.hidden_channels = hidden_channels
-    self.kernel_size = kernel_size
-    self.dilation_rate = dilation_rate
-    self.n_layers = n_layers
-    self.half_channels = channels // 2
-    self.mean_only = mean_only
-
-    self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
-    self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels)
-    self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
-    self.post.weight.data.zero_()
-    self.post.bias.data.zero_()
-
-  def forward(self, x, x_mask, g=None, reverse=False):
-    x0, x1 = torch.split(x, [self.half_channels]*2, 1)
-    h = self.pre(x0) * x_mask
-    h = self.enc(h, x_mask, g=g)
-    stats = self.post(h) * x_mask
-    if not self.mean_only:
-      m, logs = torch.split(stats, [self.half_channels]*2, 1)
-    else:
-      m = stats
-      logs = torch.zeros_like(m)
-
-    if not reverse:
-      x1 = m + x1 * torch.exp(logs) * x_mask
-      x = torch.cat([x0, x1], 1)
-      logdet = torch.sum(logs, [1,2])
-      return x, logdet
-    else:
-      x1 = (x1 - m) * torch.exp(-logs) * x_mask
-      x = torch.cat([x0, x1], 1)
-      return x
+    def __init__(self,
+                 channels,
+                 hidden_channels,
+                 kernel_size,
+                 dilation_rate,
+                 n_layers,
+                 p_dropout=0,
+                 gin_channels=0,
+                 mean_only=False):
+        assert channels % 2 == 0, "channels should be divisible by 2"
+        super().__init__()
+        self.channels = channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.half_channels = channels // 2
+        self.mean_only = mean_only
+
+        self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
+        self.enc = WN(hidden_channels,
+                      kernel_size,
+                      dilation_rate,
+                      n_layers,
+                      p_dropout=p_dropout,
+                      gin_channels=gin_channels)
+        self.post = nn.Conv1d(hidden_channels,
+                              self.half_channels * (2 - mean_only), 1)
+        self.post.weight.data.zero_()
+        self.post.bias.data.zero_()
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0) * x_mask
+        h = self.enc(h, x_mask, g=g)
+        stats = self.post(h) * x_mask
+        if not self.mean_only:
+            m, logs = torch.split(stats, [self.half_channels] * 2, 1)
+        else:
+            m = stats
+            logs = torch.zeros_like(m)
+
+        if not reverse:
+            x1 = m + x1 * torch.exp(logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            logdet = torch.sum(logs, [1, 2])
+            return x, logdet
+        else:
+            x1 = (x1 - m) * torch.exp(-logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            return x
 
 
 class ConvFlow(nn.Module):
-  def __init__(self, in_channels, filter_channels, kernel_size, n_layers, num_bins=10, tail_bound=5.0):
-    super().__init__()
-    self.in_channels = in_channels
-    self.filter_channels = filter_channels
-    self.kernel_size = kernel_size
-    self.n_layers = n_layers
-    self.num_bins = num_bins
-    self.tail_bound = tail_bound
-    self.half_channels = in_channels // 2
-
-    self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
-    self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.)
-    self.proj = nn.Conv1d(filter_channels, self.half_channels * (num_bins * 3 - 1), 1)
-    self.proj.weight.data.zero_()
-    self.proj.bias.data.zero_()
-
-  def forward(self, x, x_mask, g=None, reverse=False):
-    x0, x1 = torch.split(x, [self.half_channels]*2, 1)
-    h = self.pre(x0)
-    h = self.convs(h, x_mask, g=g)
-    h = self.proj(h) * x_mask
-
-    b, c, t = x0.shape
-    h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?]
-
-    unnormalized_widths = h[..., :self.num_bins] / math.sqrt(self.filter_channels)
-    unnormalized_heights = h[..., self.num_bins:2*self.num_bins] / math.sqrt(self.filter_channels)
-    unnormalized_derivatives = h[..., 2 * self.num_bins:]
-
-    x1, logabsdet = piecewise_rational_quadratic_transform(x1,
-        unnormalized_widths,
-        unnormalized_heights,
-        unnormalized_derivatives,
-        inverse=reverse,
-        tails='linear',
-        tail_bound=self.tail_bound
-    )
-
-    x = torch.cat([x0, x1], 1) * x_mask
-    logdet = torch.sum(logabsdet * x_mask, [1,2])
-    if not reverse:
-        return x, logdet
-    else:
-        return x
+    def __init__(self,
+                 in_channels,
+                 filter_channels,
+                 kernel_size,
+                 n_layers,
+                 num_bins=10,
+                 tail_bound=5.0):
+        super().__init__()
+        self.in_channels = in_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.num_bins = num_bins
+        self.tail_bound = tail_bound
+        self.half_channels = in_channels // 2
+
+        self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
+        self.convs = DDSConv(filter_channels,
+                             kernel_size,
+                             n_layers,
+                             p_dropout=0.)
+        self.proj = nn.Conv1d(filter_channels,
+                              self.half_channels * (num_bins * 3 - 1), 1)
+        self.proj.weight.data.zero_()
+        self.proj.bias.data.zero_()
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0)
+        h = self.convs(h, x_mask, g=g)
+        h = self.proj(h) * x_mask
+
+        b, c, t = x0.shape
+        h = h.reshape(b, c, -1, t).permute(0, 1, 3,
+                                           2)  # [b, cx?, t] -> [b, c, t, ?]
+
+        unnormalized_widths = h[..., :self.num_bins] / math.sqrt(
+            self.filter_channels)
+        unnormalized_heights = h[...,
+                                 self.num_bins:2 * self.num_bins] / math.sqrt(
+                                     self.filter_channels)
+        unnormalized_derivatives = h[..., 2 * self.num_bins:]
+
+        x1, logabsdet = piecewise_rational_quadratic_transform(
+            x1,
+            unnormalized_widths,
+            unnormalized_heights,
+            unnormalized_derivatives,
+            inverse=reverse,
+            tails='linear',
+            tail_bound=self.tail_bound)
+
+        x = torch.cat([x0, x1], 1) * x_mask
+        logdet = torch.sum(logabsdet * x_mask, [1, 2])
+        if not reverse:
+            return x, logdet
+        else:
+            return x

requirements.txt

@@ -1,5 +1,4 @@
 Cython==0.29.21
-romajitable
 librosa==0.8.0
 matplotlib==3.3.1
 numpy==1.21.6
@@ -9,7 +8,6 @@ tensorboard==2.3.0
 torch
 torchvision
 Unidecode==1.1.1
-pyopenjtalk==0.2.0
 jamo==0.4.1
 pypinyin==0.44.0
 jieba==0.42.1
@@ -17,4 +15,8 @@ cn2an==0.5.17
 jieba==0.42.1
 ipython==7.34.0
 gradio==3.4.1
-openai
+openai
+pydub
+inflect
+eng_to_ipa
+onnxruntime

text/__init__.py

@@ -1,8 +1,14 @@
 """ from https://github.com/keithito/tacotron """
 from text import cleaners
+from text.symbols import symbols
 
 
-def text_to_sequence(text, symbols, cleaner_names):
+# Mappings from symbol to numeric ID and vice versa:
+_symbol_to_id = {s: i for i, s in enumerate(symbols)}
+_id_to_symbol = {i: s for i, s in enumerate(symbols)}
+
+
+def text_to_sequence(text, cleaner_names):
   '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
     Args:
       text: string to convert to a sequence
@@ -10,8 +16,6 @@ def text_to_sequence(text, symbols, cleaner_names):
     Returns:
       List of integers corresponding to the symbols in the text
   '''
-  _symbol_to_id = {s: i for i, s in enumerate(symbols)}
-
   sequence = []
 
   clean_text = _clean_text(text, cleaner_names)
@@ -23,6 +27,26 @@ def text_to_sequence(text, symbols, cleaner_names):
   return sequence
 
 
+def cleaned_text_to_sequence(cleaned_text):
+  '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
+    Args:
+      text: string to convert to a sequence
+    Returns:
+      List of integers corresponding to the symbols in the text
+  '''
+  sequence = [_symbol_to_id[symbol] for symbol in cleaned_text if symbol in _symbol_to_id.keys()]
+  return sequence
+
+
+def sequence_to_text(sequence):
+  '''Converts a sequence of IDs back to a string'''
+  result = ''
+  for symbol_id in sequence:
+    s = _id_to_symbol[symbol_id]
+    result += s
+  return result
+
+
 def _clean_text(text, cleaner_names):
   for name in cleaner_names:
     cleaner = getattr(cleaners, name)

text/cleaners.py

@@ -1,33 +1,21 @@
 import re
-from text.japanese import japanese_to_romaji_with_accent, japanese_to_ipa, japanese_to_ipa2, japanese_to_ipa3
+from text.english import english_to_lazy_ipa, english_to_ipa2, english_to_lazy_ipa2
+from text.japanese import clean_japanese, japanese_to_romaji_with_accent, japanese_to_ipa, japanese_to_ipa2, japanese_to_ipa3
 from text.mandarin import number_to_chinese, chinese_to_bopomofo, latin_to_bopomofo, chinese_to_romaji, chinese_to_lazy_ipa, chinese_to_ipa, chinese_to_ipa2
 
-def japanese_cleaners(text):
-    from text.japanese import japanese_to_romaji_with_accent
-    text = japanese_to_romaji_with_accent(text)
-    if re.match('[A-Za-z]', text[-1]):
-        text += '.'
+def none_cleaner(text):
     return text
 
+def japanese_cleaners(text):
+    text = clean_japanese(text)
+    text = re.sub(r'([A-Za-z])$', r'\1.', text)
+    return text
 
 def japanese_cleaners2(text):
     return japanese_cleaners(text).replace('ts', 'ʦ').replace('...', '…')
 
-
-def korean_cleaners(text):
-    '''Pipeline for Korean text'''
-    from text.korean import latin_to_hangul, number_to_hangul, divide_hangul
-    text = latin_to_hangul(text)
-    text = number_to_hangul(text)
-    text = divide_hangul(text)
-    if re.match('[\u3131-\u3163]', text[-1]):
-        text += '.'
-    return text
-
-
 def chinese_cleaners(text):
     '''Pipeline for Chinese text'''
-    from text.mandarin import number_to_chinese, chinese_to_bopomofo, latin_to_bopomofo
     text = number_to_chinese(text)
     text = chinese_to_bopomofo(text)
     text = latin_to_bopomofo(text)
@@ -35,10 +23,7 @@ def chinese_cleaners(text):
         text += '。'
     return text
 
-
 def zh_ja_mixture_cleaners(text):
-    from text.mandarin import chinese_to_romaji
-    from text.japanese import japanese_to_romaji_with_accent
     chinese_texts = re.findall(r'\[ZH\].*?\[ZH\]', text)
     japanese_texts = re.findall(r'\[JA\].*?\[JA\]', text)
     for chinese_text in chinese_texts:
@@ -53,53 +38,25 @@ def zh_ja_mixture_cleaners(text):
         text += '.'
     return text
 
-
-def sanskrit_cleaners(text):
-    text = text.replace('॥', '।').replace('ॐ', 'ओम्')
-    if text[-1] != '।':
-        text += ' ।'
-    return text
-
-
-def cjks_cleaners(text):
-    from text.mandarin import chinese_to_lazy_ipa
-    from text.japanese import japanese_to_ipa
-    from text.korean import korean_to_lazy_ipa
-    from text.sanskrit import devanagari_to_ipa
-    chinese_texts = re.findall(r'\[ZH\].*?\[ZH\]', text)
-    japanese_texts = re.findall(r'\[JA\].*?\[JA\]', text)
-    korean_texts = re.findall(r'\[KO\].*?\[KO\]', text)
-    sanskrit_texts = re.findall(r'\[SA\].*?\[SA\]', text)
-    for chinese_text in chinese_texts:
-        cleaned_text = chinese_to_lazy_ipa(chinese_text[4:-4])
-        text = text.replace(chinese_text, cleaned_text+' ', 1)
-    for japanese_text in japanese_texts:
-        cleaned_text = japanese_to_ipa(japanese_text[4:-4])
-        text = text.replace(japanese_text, cleaned_text+' ', 1)
-    for korean_text in korean_texts:
-        cleaned_text = korean_to_lazy_ipa(korean_text[4:-4])
-        text = text.replace(korean_text, cleaned_text+' ', 1)
-    for sanskrit_text in sanskrit_texts:
-        cleaned_text = devanagari_to_ipa(sanskrit_text[4:-4])
-        text = text.replace(sanskrit_text, cleaned_text+' ', 1)
-    text = text[:-1]
-    if re.match(r'[^\.,!\?\-…~]', text[-1]):
-        text += '.'
-    return text
-
 def cjke_cleaners(text):
     chinese_texts = re.findall(r'\[ZH\].*?\[ZH\]', text)
     japanese_texts = re.findall(r'\[JA\].*?\[JA\]', text)
+    english_texts = re.findall(r'\[EN\].*?\[EN\]', text)
     for chinese_text in chinese_texts:
         cleaned_text = chinese_to_lazy_ipa(chinese_text[4:-4])
         cleaned_text = cleaned_text.replace(
             'ʧ', 'tʃ').replace('ʦ', 'ts').replace('ɥan', 'ɥæn')
         text = text.replace(chinese_text, cleaned_text+' ', 1)
     for japanese_text in japanese_texts:
-        cleaned_text = japanese_to_ipa(japanese_text[4:-4])
+        cleaned_text = clean_japanese(japanese_text[4:-4])
         cleaned_text = cleaned_text.replace('ʧ', 'tʃ').replace(
             'ʦ', 'ts').replace('ɥan', 'ɥæn').replace('ʥ', 'dz')
         text = text.replace(japanese_text, cleaned_text+' ', 1)
+    for english_text in english_texts:
+        cleaned_text = english_to_ipa2(english_text[4:-4])
+        cleaned_text = cleaned_text.replace('ɑ', 'a').replace(
+            'ɔ', 'o').replace('ɛ', 'e').replace('ɪ', 'i').replace('ʊ', 'u')
+        text = text.replace(english_text, cleaned_text+' ', 1)
     text = text[:-1]
     if re.match(r'[^\.,!\?\-…~]', text[-1]):
         text += '.'

text/english.py

@@ -0,0 +1,188 @@
+""" from https://github.com/keithito/tacotron """
+
+'''
+Cleaners are transformations that run over the input text at both training and eval time.
+
+Cleaners can be selected by passing a comma-delimited list of cleaner names as the "cleaners"
+hyperparameter. Some cleaners are English-specific. You'll typically want to use:
+  1. "english_cleaners" for English text
+  2. "transliteration_cleaners" for non-English text that can be transliterated to ASCII using
+     the Unidecode library (https://pypi.python.org/pypi/Unidecode)
+  3. "basic_cleaners" if you do not want to transliterate (in this case, you should also update
+     the symbols in symbols.py to match your data).
+'''
+
+
+# Regular expression matching whitespace:
+
+
+import re
+import inflect
+from unidecode import unidecode
+import eng_to_ipa as ipa
+_inflect = inflect.engine()
+_comma_number_re = re.compile(r'([0-9][0-9\,]+[0-9])')
+_decimal_number_re = re.compile(r'([0-9]+\.[0-9]+)')
+_pounds_re = re.compile(r'£([0-9\,]*[0-9]+)')
+_dollars_re = re.compile(r'\$([0-9\.\,]*[0-9]+)')
+_ordinal_re = re.compile(r'[0-9]+(st|nd|rd|th)')
+_number_re = re.compile(r'[0-9]+')
+
+# List of (regular expression, replacement) pairs for abbreviations:
+_abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [
+    ('mrs', 'misess'),
+    ('mr', 'mister'),
+    ('dr', 'doctor'),
+    ('st', 'saint'),
+    ('co', 'company'),
+    ('jr', 'junior'),
+    ('maj', 'major'),
+    ('gen', 'general'),
+    ('drs', 'doctors'),
+    ('rev', 'reverend'),
+    ('lt', 'lieutenant'),
+    ('hon', 'honorable'),
+    ('sgt', 'sergeant'),
+    ('capt', 'captain'),
+    ('esq', 'esquire'),
+    ('ltd', 'limited'),
+    ('col', 'colonel'),
+    ('ft', 'fort'),
+]]
+
+
+# List of (ipa, lazy ipa) pairs:
+_lazy_ipa = [(re.compile('%s' % x[0]), x[1]) for x in [
+    ('r', 'ɹ'),
+    ('æ', 'e'),
+    ('ɑ', 'a'),
+    ('ɔ', 'o'),
+    ('ð', 'z'),
+    ('θ', 's'),
+    ('ɛ', 'e'),
+    ('ɪ', 'i'),
+    ('ʊ', 'u'),
+    ('ʒ', 'ʥ'),
+    ('ʤ', 'ʥ'),
+    ('ˈ', '↓'),
+]]
+
+# List of (ipa, lazy ipa2) pairs:
+_lazy_ipa2 = [(re.compile('%s' % x[0]), x[1]) for x in [
+    ('r', 'ɹ'),
+    ('ð', 'z'),
+    ('θ', 's'),
+    ('ʒ', 'ʑ'),
+    ('ʤ', 'dʑ'),
+    ('ˈ', '↓'),
+]]
+
+# List of (ipa, ipa2) pairs
+_ipa_to_ipa2 = [(re.compile('%s' % x[0]), x[1]) for x in [
+    ('r', 'ɹ'),
+    ('ʤ', 'dʒ'),
+    ('ʧ', 'tʃ')
+]]
+
+
+def expand_abbreviations(text):
+    for regex, replacement in _abbreviations:
+        text = re.sub(regex, replacement, text)
+    return text
+
+
+def collapse_whitespace(text):
+    return re.sub(r'\s+', ' ', text)
+
+
+def _remove_commas(m):
+    return m.group(1).replace(',', '')
+
+
+def _expand_decimal_point(m):
+    return m.group(1).replace('.', ' point ')
+
+
+def _expand_dollars(m):
+    match = m.group(1)
+    parts = match.split('.')
+    if len(parts) > 2:
+        return match + ' dollars'  # Unexpected format
+    dollars = int(parts[0]) if parts[0] else 0
+    cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
+    if dollars and cents:
+        dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+        cent_unit = 'cent' if cents == 1 else 'cents'
+        return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit)
+    elif dollars:
+        dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+        return '%s %s' % (dollars, dollar_unit)
+    elif cents:
+        cent_unit = 'cent' if cents == 1 else 'cents'
+        return '%s %s' % (cents, cent_unit)
+    else:
+        return 'zero dollars'
+
+
+def _expand_ordinal(m):
+    return _inflect.number_to_words(m.group(0))
+
+
+def _expand_number(m):
+    num = int(m.group(0))
+    if num > 1000 and num < 3000:
+        if num == 2000:
+            return 'two thousand'
+        elif num > 2000 and num < 2010:
+            return 'two thousand ' + _inflect.number_to_words(num % 100)
+        elif num % 100 == 0:
+            return _inflect.number_to_words(num // 100) + ' hundred'
+        else:
+            return _inflect.number_to_words(num, andword='', zero='oh', group=2).replace(', ', ' ')
+    else:
+        return _inflect.number_to_words(num, andword='')
+
+
+def normalize_numbers(text):
+    text = re.sub(_comma_number_re, _remove_commas, text)
+    text = re.sub(_pounds_re, r'\1 pounds', text)
+    text = re.sub(_dollars_re, _expand_dollars, text)
+    text = re.sub(_decimal_number_re, _expand_decimal_point, text)
+    text = re.sub(_ordinal_re, _expand_ordinal, text)
+    text = re.sub(_number_re, _expand_number, text)
+    return text
+
+
+def mark_dark_l(text):
+    return re.sub(r'l([^aeiouæɑɔəɛɪʊ ]*(?: |$))', lambda x: 'ɫ'+x.group(1), text)
+
+
+def english_to_ipa(text):
+    text = unidecode(text).lower()
+    text = expand_abbreviations(text)
+    text = normalize_numbers(text)
+    phonemes = ipa.convert(text)
+    phonemes = collapse_whitespace(phonemes)
+    return phonemes
+
+
+def english_to_lazy_ipa(text):
+    text = english_to_ipa(text)
+    for regex, replacement in _lazy_ipa:
+        text = re.sub(regex, replacement, text)
+    return text
+
+
+def english_to_ipa2(text):
+    text = english_to_ipa(text)
+    text = mark_dark_l(text)
+    for regex, replacement in _ipa_to_ipa2:
+        text = re.sub(regex, replacement, text)
+    return text.replace('...', '…')
+
+
+def english_to_lazy_ipa2(text):
+    text = english_to_ipa(text)
+    for regex, replacement in _lazy_ipa2:
+        text = re.sub(regex, replacement, text)
+    return text

text/japanese.py

@@ -1,6 +1,18 @@
 import re
 from unidecode import unidecode
-import pyopenjtalk
+from unidecode import unidecode
+import ctypes
+
+dll = ctypes.cdll.LoadLibrary('cleaners/JapaneseCleaner.dll')
+dll.CreateOjt.restype = ctypes.c_uint64 
+dll.PluginMain.restype = ctypes.c_uint64 
+floder = ctypes.create_unicode_buffer("cleaners")
+dll.CreateOjt(floder)
+
+def clean_japanese(text):
+    input_wchar_pointer = ctypes.create_unicode_buffer(text)
+    result = ctypes.wstring_at(dll.PluginMain(input_wchar_pointer))
+    return result
 
 
 # Regular expression matching Japanese without punctuation marks:

text/symbols.py

@@ -0,0 +1,67 @@
+'''
+Defines the set of symbols used in text input to the model.
+'''
+_pad        = '_'
+_punctuation = ',.!?-~…'
+_letters = 'AEINOQUabdefghijklmnoprstuvwyzʃʧʦɯɹəɥ⁼ʰ`→↓↑ '
+'''
+# japanese_cleaners2
+_pad        = '_'
+_punctuation = ',.!?-~…'
+_letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧʦ↓↑ '
+'''
+
+'''# korean_cleaners
+_pad        = '_'
+_punctuation = ',.!?…~'
+_letters = 'ㄱㄴㄷㄹㅁㅂㅅㅇㅈㅊㅋㅌㅍㅎㄲㄸㅃㅆㅉㅏㅓㅗㅜㅡㅣㅐㅔ '
+'''
+
+'''# chinese_cleaners
+_pad        = '_'
+_punctuation = '，。！？—…'
+_letters = 'ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩˉˊˇˋ˙ '
+'''
+
+
+'''# sanskrit_cleaners
+_pad        = '_'
+_punctuation = '।'
+_letters = 'ँंःअआइईउऊऋएऐओऔकखगघङचछजझञटठडढणतथदधनपफबभमयरलळवशषसहऽािीुूृॄेैोौ्ॠॢ '
+'''
+
+'''# cjks_cleaners
+_pad        = '_'
+_punctuation = ',.!?-~…'
+_letters = 'NQabdefghijklmnopstuvwxyzʃʧʥʦɯɹəɥçɸɾβŋɦː⁼ʰ`^#*=→↓↑ '
+'''
+
+'''# thai_cleaners
+_pad        = '_'
+_punctuation = '.!? '
+_letters = 'กขฃคฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลวศษสหฬอฮฯะัาำิีึืุูเแโใไๅๆ็่้๊๋์'
+'''
+
+'''# cjke_cleaners2
+_pad        = '_'
+_punctuation = ',.!?-~…'
+_letters = 'NQabdefghijklmnopstuvwxyzɑæʃʑçɯɪɔɛɹðəɫɥɸʊɾʒθβŋɦ⁼ʰ`^#*=ˈˌ→↓↑ '
+'''
+
+'''# shanghainese_cleaners
+_pad        = '_'
+_punctuation = ',.!?…'
+_letters = 'abdfghiklmnopstuvyzøŋȵɑɔɕəɤɦɪɿʑʔʰ̩̃ᴀᴇ15678 '
+'''
+
+'''# chinese_dialect_cleaners
+_pad        = '_'
+_punctuation = ',.!?~…─'
+_letters = '#Nabdefghijklmnoprstuvwxyzæçøŋœȵɐɑɒɓɔɕɗɘəɚɛɜɣɤɦɪɭɯɵɷɸɻɾɿʂʅʊʋʌʏʑʔʦʮʰʷˀː˥˦˧˨˩̥̩̃̚αᴀᴇ↑↓∅ⱼ '
+'''
+
+# Export all symbols:
+symbols = [_pad] + list(_punctuation) + list(_letters)
+
+# Special symbol ids
+SPACE_ID = symbols.index(" ")

train.py

@@ -0,0 +1,328 @@
+import os
+
+import torch
+from torch.nn import functional as F
+from torch.utils.data import DataLoader
+from torch.utils.tensorboard import SummaryWriter
+import torch.multiprocessing as mp
+import torch.distributed as dist
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.cuda.amp import autocast, GradScaler
+
+import commons
+import utils
+from data_utils import (TextAudioSpeakerLoader, TextAudioSpeakerCollate,
+                        DistributedBucketSampler)
+from models import (
+    SynthesizerTrn,
+    MultiPeriodDiscriminator,
+)
+from losses import (generator_loss, discriminator_loss, feature_loss, kl_loss)
+from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
+
+torch.backends.cudnn.benchmark = True
+global_step = 0
+
+
+def main():
+    """Assume Single Node Multi GPUs Training Only"""
+    assert torch.cuda.is_available(), "CPU training is not allowed."
+
+    n_gpus = torch.cuda.device_count()
+    hps = utils.get_hparams()
+    mp.spawn(run, nprocs=n_gpus, args=(
+        n_gpus,
+        hps,
+    ))
+
+
+def run(rank, n_gpus, hps):
+    global global_step
+    if rank == 0:
+        logger = utils.get_logger(hps.model_dir)
+        logger.info(hps)
+        utils.check_git_hash(hps.model_dir)
+        writer = SummaryWriter(log_dir=hps.model_dir)
+        writer_eval = SummaryWriter(
+            log_dir=os.path.join(hps.model_dir, "eval"))
+
+    dist.init_process_group(backend='nccl',
+                            init_method='env://',
+                            world_size=n_gpus,
+                            rank=rank)
+    torch.manual_seed(hps.train.seed)
+    torch.cuda.set_device(rank)
+    train_dataset = TextAudioSpeakerLoader(hps.data.training_files, hps.data)
+    train_sampler = DistributedBucketSampler(
+        train_dataset,
+        hps.train.batch_size, [32, 300, 400, 500, 600, 700, 800, 900, 1000],
+        num_replicas=n_gpus,
+        rank=rank,
+        shuffle=True)
+    collate_fn = TextAudioSpeakerCollate()
+    train_loader = DataLoader(train_dataset,
+                              num_workers=8,
+                              shuffle=False,
+                              pin_memory=True,
+                              collate_fn=collate_fn,
+                              batch_sampler=train_sampler)
+    if rank == 0:
+        eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files,
+                                              hps.data)
+        eval_loader = DataLoader(eval_dataset,
+                                 num_workers=8,
+                                 shuffle=False,
+                                 batch_size=hps.train.batch_size,
+                                 pin_memory=True,
+                                 drop_last=False,
+                                 collate_fn=collate_fn)
+
+    net_g = SynthesizerTrn(hps.data.num_phones,
+                           hps.data.filter_length // 2 + 1,
+                           hps.train.segment_size // hps.data.hop_length,
+                           n_speakers=hps.data.n_speakers,
+                           **hps.model).cuda(rank)
+    net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank)
+    optim_g = torch.optim.AdamW(net_g.parameters(),
+                                hps.train.learning_rate,
+                                betas=hps.train.betas,
+                                eps=hps.train.eps)
+    optim_d = torch.optim.AdamW(net_d.parameters(),
+                                hps.train.learning_rate,
+                                betas=hps.train.betas,
+                                eps=hps.train.eps)
+    net_g = DDP(net_g, device_ids=[rank])
+    net_d = DDP(net_d, device_ids=[rank])
+
+    try:
+        _, _, _, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g,
+            optim_g)
+        _, _, _, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d,
+            optim_d)
+        global_step = (epoch_str - 1) * len(train_loader)
+    except Exception as e:
+        epoch_str = 1
+        global_step = 0
+
+    scheduler_g = torch.optim.lr_scheduler.ExponentialLR(
+        optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2)
+    scheduler_d = torch.optim.lr_scheduler.ExponentialLR(
+        optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2)
+
+    scaler = GradScaler(enabled=hps.train.fp16_run)
+
+    for epoch in range(epoch_str, hps.train.epochs + 1):
+        if rank == 0:
+            train_and_evaluate(rank, epoch, hps, [net_g, net_d],
+                               [optim_g, optim_d], [scheduler_g, scheduler_d],
+                               scaler, [train_loader, eval_loader], logger,
+                               [writer, writer_eval])
+        else:
+            train_and_evaluate(rank, epoch, hps, [net_g, net_d],
+                               [optim_g, optim_d], [scheduler_g, scheduler_d],
+                               scaler, [train_loader, None], None, None)
+        scheduler_g.step()
+        scheduler_d.step()
+
+
+def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler,
+                       loaders, logger, writers):
+    net_g, net_d = nets
+    optim_g, optim_d = optims
+    scheduler_g, scheduler_d = schedulers
+    train_loader, eval_loader = loaders
+    if writers is not None:
+        writer, writer_eval = writers
+
+    train_loader.batch_sampler.set_epoch(epoch)
+    global global_step
+
+    net_g.train()
+    net_d.train()
+    for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths,
+                    speakers) in enumerate(train_loader):
+        x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(
+            rank, non_blocking=True)
+        spec, spec_lengths = spec.cuda(
+            rank, non_blocking=True), spec_lengths.cuda(rank,
+                                                        non_blocking=True)
+        y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(
+            rank, non_blocking=True)
+        speakers = speakers.cuda(rank, non_blocking=True)
+
+        with autocast(enabled=hps.train.fp16_run):
+            y_hat, l_length, attn, ids_slice, x_mask, z_mask, (
+                z, z_p, m_p, logs_p, m_q,
+                logs_q) = net_g(x, x_lengths, spec, spec_lengths, speakers)
+
+            mel = spec_to_mel_torch(spec, hps.data.filter_length,
+                                    hps.data.n_mel_channels,
+                                    hps.data.sampling_rate, hps.data.mel_fmin,
+                                    hps.data.mel_fmax)
+            y_mel = commons.slice_segments(
+                mel, ids_slice, hps.train.segment_size // hps.data.hop_length)
+            y_hat_mel = mel_spectrogram_torch(
+                y_hat.squeeze(1), hps.data.filter_length,
+                hps.data.n_mel_channels, hps.data.sampling_rate,
+                hps.data.hop_length, hps.data.win_length, hps.data.mel_fmin,
+                hps.data.mel_fmax)
+
+            y = commons.slice_segments(y, ids_slice * hps.data.hop_length,
+                                       hps.train.segment_size)  # slice
+
+            # Discriminator
+            y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
+            with autocast(enabled=False):
+                loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(
+                    y_d_hat_r, y_d_hat_g)
+                loss_disc_all = loss_disc
+        optim_d.zero_grad()
+        scaler.scale(loss_disc_all).backward()
+        scaler.unscale_(optim_d)
+        grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
+        scaler.step(optim_d)
+
+        with autocast(enabled=hps.train.fp16_run):
+            # Generator
+            y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat)
+            with autocast(enabled=False):
+                loss_dur = torch.sum(l_length.float())
+                loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
+                loss_kl = kl_loss(z_p, logs_q, m_p, logs_p,
+                                  z_mask) * hps.train.c_kl
+
+                loss_fm = feature_loss(fmap_r, fmap_g)
+                loss_gen, losses_gen = generator_loss(y_d_hat_g)
+                loss_gen_all = loss_gen + loss_fm + loss_mel + loss_dur + loss_kl
+        optim_g.zero_grad()
+        scaler.scale(loss_gen_all).backward()
+        scaler.unscale_(optim_g)
+        grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
+        scaler.step(optim_g)
+        scaler.update()
+
+        if rank == 0:
+            if global_step % hps.train.log_interval == 0:
+                lr = optim_g.param_groups[0]['lr']
+                losses = [
+                    loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl
+                ]
+                logger.info('Train Epoch: {} [{:.0f}%]'.format(
+                    epoch, 100. * batch_idx / len(train_loader)))
+                logger.info([x.item() for x in losses] + [global_step, lr])
+
+                scalar_dict = {
+                    "loss/g/total": loss_gen_all,
+                    "loss/d/total": loss_disc_all,
+                    "learning_rate": lr,
+                    "grad_norm_d": grad_norm_d,
+                    "grad_norm_g": grad_norm_g
+                }
+                scalar_dict.update({
+                    "loss/g/fm": loss_fm,
+                    "loss/g/mel": loss_mel,
+                    "loss/g/dur": loss_dur,
+                    "loss/g/kl": loss_kl
+                })
+
+                scalar_dict.update({
+                    "loss/g/{}".format(i): v
+                    for i, v in enumerate(losses_gen)
+                })
+                scalar_dict.update({
+                    "loss/d_r/{}".format(i): v
+                    for i, v in enumerate(losses_disc_r)
+                })
+                scalar_dict.update({
+                    "loss/d_g/{}".format(i): v
+                    for i, v in enumerate(losses_disc_g)
+                })
+                image_dict = {
+                    "slice/mel_org":
+                    utils.plot_spectrogram_to_numpy(
+                        y_mel[0].data.cpu().numpy()),
+                    "slice/mel_gen":
+                    utils.plot_spectrogram_to_numpy(
+                        y_hat_mel[0].data.cpu().numpy()),
+                    "all/mel":
+                    utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()),
+                    "all/attn":
+                    utils.plot_alignment_to_numpy(attn[0,
+                                                       0].data.cpu().numpy())
+                }
+                utils.summarize(writer=writer,
+                                global_step=global_step,
+                                images=image_dict,
+                                scalars=scalar_dict)
+
+            if global_step % hps.train.eval_interval == 0:
+                evaluate(hps, net_g, eval_loader, writer_eval)
+                utils.save_checkpoint(
+                    net_g, optim_g, hps.train.learning_rate, epoch,
+                    os.path.join(hps.model_dir,
+                                 "G_{}.pth".format(global_step)))
+                utils.save_checkpoint(
+                    net_d, optim_d, hps.train.learning_rate, epoch,
+                    os.path.join(hps.model_dir,
+                                 "D_{}.pth".format(global_step)))
+        global_step += 1
+
+    if rank == 0:
+        logger.info('====> Epoch: {}'.format(epoch))
+
+
+def evaluate(hps, generator, eval_loader, writer_eval):
+    generator.eval()
+    with torch.no_grad():
+        for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths,
+                        speakers) in enumerate(eval_loader):
+            x, x_lengths = x.cuda(0), x_lengths.cuda(0)
+            spec, spec_lengths = spec.cuda(0), spec_lengths.cuda(0)
+            y, y_lengths = y.cuda(0), y_lengths.cuda(0)
+            speakers = speakers.cuda(0)
+
+            # remove else
+            x = x[:1]
+            x_lengths = x_lengths[:1]
+            spec = spec[:1]
+            spec_lengths = spec_lengths[:1]
+            y = y[:1]
+            y_lengths = y_lengths[:1]
+            speakers = speakers[:1]
+            break
+        y_hat, attn, mask, *_ = generator.module.infer(x,
+                                                       x_lengths,
+                                                       speakers,
+                                                       max_len=1000)
+        y_hat_lengths = mask.sum([1, 2]).long() * hps.data.hop_length
+
+        mel = spec_to_mel_torch(spec, hps.data.filter_length,
+                                hps.data.n_mel_channels,
+                                hps.data.sampling_rate, hps.data.mel_fmin,
+                                hps.data.mel_fmax)
+        y_hat_mel = mel_spectrogram_torch(
+            y_hat.squeeze(1).float(), hps.data.filter_length,
+            hps.data.n_mel_channels, hps.data.sampling_rate,
+            hps.data.hop_length, hps.data.win_length, hps.data.mel_fmin,
+            hps.data.mel_fmax)
+    image_dict = {
+        "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy())
+    }
+    audio_dict = {"gen/audio": y_hat[0, :, :y_hat_lengths[0]]}
+    if global_step == 0:
+        image_dict.update(
+            {"gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())})
+        audio_dict.update({"gt/audio": y[0, :, :y_lengths[0]]})
+
+    utils.summarize(writer=writer_eval,
+                    global_step=global_step,
+                    images=image_dict,
+                    audios=audio_dict,
+                    audio_sampling_rate=hps.data.sampling_rate)
+    generator.train()
+
+
+if __name__ == "__main__":
+    main()

transforms.py

@@ -1,67 +1,60 @@
+import numpy as np
 import torch
 from torch.nn import functional as F
 
-import numpy as np
-
-
 DEFAULT_MIN_BIN_WIDTH = 1e-3
 DEFAULT_MIN_BIN_HEIGHT = 1e-3
 DEFAULT_MIN_DERIVATIVE = 1e-3
 
 
-def piecewise_rational_quadratic_transform(inputs, 
-                                           unnormalized_widths,
-                                           unnormalized_heights,
-                                           unnormalized_derivatives,
-                                           inverse=False,
-                                           tails=None, 
-                                           tail_bound=1.,
-                                           min_bin_width=DEFAULT_MIN_BIN_WIDTH,
-                                           min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
-                                           min_derivative=DEFAULT_MIN_DERIVATIVE):
+def piecewise_rational_quadratic_transform(
+        inputs,
+        unnormalized_widths,
+        unnormalized_heights,
+        unnormalized_derivatives,
+        inverse=False,
+        tails=None,
+        tail_bound=1.,
+        min_bin_width=DEFAULT_MIN_BIN_WIDTH,
+        min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
+        min_derivative=DEFAULT_MIN_DERIVATIVE):
 
     if tails is None:
         spline_fn = rational_quadratic_spline
         spline_kwargs = {}
     else:
         spline_fn = unconstrained_rational_quadratic_spline
-        spline_kwargs = {
-            'tails': tails,
-            'tail_bound': tail_bound
-        }
+        spline_kwargs = {'tails': tails, 'tail_bound': tail_bound}
 
     outputs, logabsdet = spline_fn(
-            inputs=inputs,
-            unnormalized_widths=unnormalized_widths,
-            unnormalized_heights=unnormalized_heights,
-            unnormalized_derivatives=unnormalized_derivatives,
-            inverse=inverse,
-            min_bin_width=min_bin_width,
-            min_bin_height=min_bin_height,
-            min_derivative=min_derivative,
-            **spline_kwargs
-    )
+        inputs=inputs,
+        unnormalized_widths=unnormalized_widths,
+        unnormalized_heights=unnormalized_heights,
+        unnormalized_derivatives=unnormalized_derivatives,
+        inverse=inverse,
+        min_bin_width=min_bin_width,
+        min_bin_height=min_bin_height,
+        min_derivative=min_derivative,
+        **spline_kwargs)
     return outputs, logabsdet
 
 
 def searchsorted(bin_locations, inputs, eps=1e-6):
-    bin_locations[..., -1] += eps
-    return torch.sum(
-        inputs[..., None] >= bin_locations,
-        dim=-1
-    ) - 1
-
-
-def unconstrained_rational_quadratic_spline(inputs,
-                                            unnormalized_widths,
-                                            unnormalized_heights,
-                                            unnormalized_derivatives,
-                                            inverse=False,
-                                            tails='linear',
-                                            tail_bound=1.,
-                                            min_bin_width=DEFAULT_MIN_BIN_WIDTH,
-                                            min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
-                                            min_derivative=DEFAULT_MIN_DERIVATIVE):
+    bin_locations[..., bin_locations.size(-1) - 1] += eps
+    return torch.sum(inputs[..., None] >= bin_locations, dim=-1) - 1
+
+
+def unconstrained_rational_quadratic_spline(
+        inputs,
+        unnormalized_widths,
+        unnormalized_heights,
+        unnormalized_derivatives,
+        inverse=False,
+        tails='linear',
+        tail_bound=1.,
+        min_bin_width=DEFAULT_MIN_BIN_WIDTH,
+        min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
+        min_derivative=DEFAULT_MIN_DERIVATIVE):
     inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)
     outside_interval_mask = ~inside_interval_mask
 
@@ -72,33 +65,41 @@ def unconstrained_rational_quadratic_spline(inputs,
         unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1))
         constant = np.log(np.exp(1 - min_derivative) - 1)
         unnormalized_derivatives[..., 0] = constant
-        unnormalized_derivatives[..., -1] = constant
+        unnormalized_derivatives[..., unnormalized_derivatives.size(-1) - 1] = constant
 
         outputs[outside_interval_mask] = inputs[outside_interval_mask]
         logabsdet[outside_interval_mask] = 0
     else:
         raise RuntimeError('{} tails are not implemented.'.format(tails))
 
-    outputs[inside_interval_mask], logabsdet[inside_interval_mask] = rational_quadratic_spline(
-        inputs=inputs[inside_interval_mask],
-        unnormalized_widths=unnormalized_widths[inside_interval_mask, :],
-        unnormalized_heights=unnormalized_heights[inside_interval_mask, :],
-        unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :],
-        inverse=inverse,
-        left=-tail_bound, right=tail_bound, bottom=-tail_bound, top=tail_bound,
-        min_bin_width=min_bin_width,
-        min_bin_height=min_bin_height,
-        min_derivative=min_derivative
-    )
+    outputs[inside_interval_mask], logabsdet[
+        inside_interval_mask] = rational_quadratic_spline(
+            inputs=inputs[inside_interval_mask],
+            unnormalized_widths=unnormalized_widths[inside_interval_mask, :],
+            unnormalized_heights=unnormalized_heights[inside_interval_mask, :],
+            unnormalized_derivatives=unnormalized_derivatives[
+                inside_interval_mask, :],
+            inverse=inverse,
+            left=-tail_bound,
+            right=tail_bound,
+            bottom=-tail_bound,
+            top=tail_bound,
+            min_bin_width=min_bin_width,
+            min_bin_height=min_bin_height,
+            min_derivative=min_derivative)
 
     return outputs, logabsdet
 
+
 def rational_quadratic_spline(inputs,
                               unnormalized_widths,
                               unnormalized_heights,
                               unnormalized_derivatives,
                               inverse=False,
-                              left=0., right=1., bottom=0., top=1.,
+                              left=0.,
+                              right=1.,
+                              bottom=0.,
+                              top=1.,
                               min_bin_width=DEFAULT_MIN_BIN_WIDTH,
                               min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
                               min_derivative=DEFAULT_MIN_DERIVATIVE):
@@ -118,7 +119,7 @@ def rational_quadratic_spline(inputs,
     cumwidths = F.pad(cumwidths, pad=(1, 0), mode='constant', value=0.0)
     cumwidths = (right - left) * cumwidths + left
     cumwidths[..., 0] = left
-    cumwidths[..., -1] = right
+    cumwidths[..., cumwidths.size(-1) - 1] = right
     widths = cumwidths[..., 1:] - cumwidths[..., :-1]
 
     derivatives = min_derivative + F.softplus(unnormalized_derivatives)
@@ -129,7 +130,7 @@ def rational_quadratic_spline(inputs,
     cumheights = F.pad(cumheights, pad=(1, 0), mode='constant', value=0.0)
     cumheights = (top - bottom) * cumheights + bottom
     cumheights[..., 0] = bottom
-    cumheights[..., -1] = top
+    cumheights[..., cumheights.size(-1) - 1] = top
     heights = cumheights[..., 1:] - cumheights[..., :-1]
 
     if inverse:
@@ -145,20 +146,20 @@ def rational_quadratic_spline(inputs,
     input_delta = delta.gather(-1, bin_idx)[..., 0]
 
     input_derivatives = derivatives.gather(-1, bin_idx)[..., 0]
-    input_derivatives_plus_one = derivatives[..., 1:].gather(-1, bin_idx)[..., 0]
+    input_derivatives_plus_one = derivatives[..., 1:].gather(-1, bin_idx)[...,
+                                                                          0]
 
     input_heights = heights.gather(-1, bin_idx)[..., 0]
 
     if inverse:
-        a = (((inputs - input_cumheights) * (input_derivatives
-                                             + input_derivatives_plus_one
-                                             - 2 * input_delta)
-              + input_heights * (input_delta - input_derivatives)))
-        b = (input_heights * input_derivatives
-             - (inputs - input_cumheights) * (input_derivatives
-                                              + input_derivatives_plus_one
-                                              - 2 * input_delta))
-        c = - input_delta * (inputs - input_cumheights)
+        a = (
+            ((inputs - input_cumheights) *
+             (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
+             + input_heights * (input_delta - input_derivatives)))
+        b = (
+            input_heights * input_derivatives - (inputs - input_cumheights) *
+            (input_derivatives + input_derivatives_plus_one - 2 * input_delta))
+        c = -input_delta * (inputs - input_cumheights)
 
         discriminant = b.pow(2) - 4 * a * c
         assert (discriminant >= 0).all()
@@ -167,27 +168,33 @@ def rational_quadratic_spline(inputs,
         outputs = root * input_bin_widths + input_cumwidths
 
         theta_one_minus_theta = root * (1 - root)
-        denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta)
-                                     * theta_one_minus_theta)
-        derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * root.pow(2)
-                                                     + 2 * input_delta * theta_one_minus_theta
-                                                     + input_derivatives * (1 - root).pow(2))
-        logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
+        denominator = input_delta + (
+            (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
+            * theta_one_minus_theta)
+        derivative_numerator = input_delta.pow(2) * (
+            input_derivatives_plus_one * root.pow(2) +
+            2 * input_delta * theta_one_minus_theta + input_derivatives *
+            (1 - root).pow(2))
+        logabsdet = torch.log(
+            derivative_numerator) - 2 * torch.log(denominator)
 
         return outputs, -logabsdet
     else:
         theta = (inputs - input_cumwidths) / input_bin_widths
         theta_one_minus_theta = theta * (1 - theta)
 
-        numerator = input_heights * (input_delta * theta.pow(2)
-                                     + input_derivatives * theta_one_minus_theta)
-        denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta)
-                                     * theta_one_minus_theta)
+        numerator = input_heights * (input_delta * theta.pow(2) +
+                                     input_derivatives * theta_one_minus_theta)
+        denominator = input_delta + (
+            (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
+            * theta_one_minus_theta)
         outputs = input_cumheights + numerator / denominator
 
-        derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * theta.pow(2)
-                                                     + 2 * input_delta * theta_one_minus_theta
-                                                     + input_derivatives * (1 - theta).pow(2))
-        logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
+        derivative_numerator = input_delta.pow(2) * (
+            input_derivatives_plus_one * theta.pow(2) +
+            2 * input_delta * theta_one_minus_theta + input_derivatives *
+            (1 - theta).pow(2))
+        logabsdet = torch.log(
+            derivative_numerator) - 2 * torch.log(denominator)
 
         return outputs, logabsdet

utils.py

@@ -1,8 +1,278 @@
+import argparse
+import glob
+import json
 import logging
-from json import loads
-from torch import load, FloatTensor
-from numpy import float32
-import librosa
+import os
+import subprocess
+import sys
+
+import numpy as np
+from scipy.io.wavfile import read
+import torch
+
+MATPLOTLIB_FLAG = False
+
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging
+
+
+def load_checkpoint(checkpoint_path, model, optimizer=None):
+    assert os.path.isfile(checkpoint_path)
+    checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
+    iteration = checkpoint_dict['iteration']
+    learning_rate = checkpoint_dict['learning_rate']
+    if optimizer is not None:
+        optimizer.load_state_dict(checkpoint_dict['optimizer'])
+    saved_state_dict = checkpoint_dict['model']
+    if hasattr(model, 'module'):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    new_state_dict = {}
+    for k, v in state_dict.items():
+        try:
+            new_state_dict[k] = saved_state_dict[k]
+        except Exception as e:
+            logger.info("%s is not in the checkpoint" % k)
+            new_state_dict[k] = v
+    if hasattr(model, 'module'):
+        model.module.load_state_dict(new_state_dict)
+    else:
+        model.load_state_dict(new_state_dict)
+    logger.info("Loaded checkpoint '{}' (iteration {})".format(
+        checkpoint_path, iteration))
+    return model, optimizer, learning_rate, iteration
+
+
+def save_checkpoint(model, optimizer, learning_rate, iteration,
+                    checkpoint_path):
+    logger.info(
+        "Saving model and optimizer state at iteration {} to {}".format(
+            iteration, checkpoint_path))
+    if hasattr(model, 'module'):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    torch.save(
+        {
+            'model': state_dict,
+            'iteration': iteration,
+            'optimizer': optimizer.state_dict(),
+            'learning_rate': learning_rate
+        }, checkpoint_path)
+
+
+def summarize(
+        writer,
+        global_step,
+        scalars={},  # noqa
+        histograms={},  # noqa
+        images={},  # noqa
+        audios={},  # noqa
+        audio_sampling_rate=22050):
+    for k, v in scalars.items():
+        writer.add_scalar(k, v, global_step)
+    for k, v in histograms.items():
+        writer.add_histogram(k, v, global_step)
+    for k, v in images.items():
+        writer.add_image(k, v, global_step, dataformats='HWC')
+    for k, v in audios.items():
+        writer.add_audio(k, v, global_step, audio_sampling_rate)
+
+
+def latest_checkpoint_path(dir_path, regex="G_*.pth"):
+    f_list = glob.glob(os.path.join(dir_path, regex))
+    f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
+    x = f_list[-1]
+    print(x)
+    return x
+
+
+def plot_spectrogram_to_numpy(spectrogram):
+    global MATPLOTLIB_FLAG
+    if not MATPLOTLIB_FLAG:
+        import matplotlib
+        matplotlib.use("Agg")
+        MATPLOTLIB_FLAG = True
+        mpl_logger = logging.getLogger('matplotlib')
+        mpl_logger.setLevel(logging.WARNING)
+    import matplotlib.pylab as plt
+    import numpy as np
+
+    fig, ax = plt.subplots(figsize=(10, 2))
+    im = ax.imshow(spectrogram,
+                   aspect="auto",
+                   origin="lower",
+                   interpolation='none')
+    plt.colorbar(im, ax=ax)
+    plt.xlabel("Frames")
+    plt.ylabel("Channels")
+    plt.tight_layout()
+
+    fig.canvas.draw()
+    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
+    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3, ))
+    plt.close()
+    return data
+
+
+def plot_alignment_to_numpy(alignment, info=None):
+    global MATPLOTLIB_FLAG
+    if not MATPLOTLIB_FLAG:
+        import matplotlib
+        matplotlib.use("Agg")
+        MATPLOTLIB_FLAG = True
+        mpl_logger = logging.getLogger('matplotlib')
+        mpl_logger.setLevel(logging.WARNING)
+    import matplotlib.pylab as plt
+    import numpy as np
+
+    fig, ax = plt.subplots(figsize=(6, 4))
+    im = ax.imshow(alignment.transpose(),
+                   aspect='auto',
+                   origin='lower',
+                   interpolation='none')
+    fig.colorbar(im, ax=ax)
+    xlabel = 'Decoder timestep'
+    if info is not None:
+        xlabel += '\n\n' + info
+    plt.xlabel(xlabel)
+    plt.ylabel('Encoder timestep')
+    plt.tight_layout()
+
+    fig.canvas.draw()
+    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
+    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3, ))
+    plt.close()
+    return data
+
+
+def load_wav_to_torch(full_path):
+    sampling_rate, data = read(full_path)
+    return torch.FloatTensor(data.astype(np.float32)), sampling_rate
+
+
+def load_filepaths_and_text(filename, split="|"):
+    with open(filename, encoding='utf-8') as f:
+        filepaths_and_text = [line.strip().split(split) for line in f]
+    return filepaths_and_text
+
+
+def get_hparams(init=True):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c',
+                        '--config',
+                        type=str,
+                        default="./configs/base.json",
+                        help='JSON file for configuration')
+    parser.add_argument('-m',
+                        '--model',
+                        type=str,
+                        required=True,
+                        help='Model name')
+    parser.add_argument('--train_data',
+                        type=str,
+                        required=True,
+                        help='train data')
+    parser.add_argument('--val_data', type=str, required=True, help='val data')
+    parser.add_argument('--phone_table',
+                        type=str,
+                        required=True,
+                        help='phone table')
+    parser.add_argument('--speaker_table',
+                        type=str,
+                        default=None,
+                        help='speaker table, required for multiple speakers')
+
+    args = parser.parse_args()
+    model_dir = args.model
+
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+
+    config_path = args.config
+    config_save_path = os.path.join(model_dir, "config.json")
+    if init:
+        with open(config_path, "r", encoding='utf8') as f:
+            data = f.read()
+        with open(config_save_path, "w", encoding='utf8') as f:
+            f.write(data)
+    else:
+        with open(config_save_path, "r", encoding='utf8') as f:
+            data = f.read()
+    config = json.loads(data)
+    config['data']['training_files'] = args.train_data
+    config['data']['validation_files'] = args.val_data
+    config['data']['phone_table'] = args.phone_table
+    # 0 is kept for blank
+    config['data']['num_phones'] = len(open(args.phone_table).readlines()) + 1
+    if args.speaker_table is not None:
+        config['data']['speaker_table'] = args.speaker_table
+        # 0 is kept for unknown speaker
+        config['data']['n_speakers'] = len(
+            open(args.speaker_table).readlines()) + 1
+    else:
+        config['data']['n_speakers'] = 0
+
+    hparams = HParams(**config)
+    hparams.model_dir = model_dir
+    return hparams
+
+
+def get_hparams_from_dir(model_dir):
+    config_save_path = os.path.join(model_dir, "config.json")
+    with open(config_save_path, "r") as f:
+        data = f.read()
+    config = json.loads(data)
+
+    hparams = HParams(**config)
+    hparams.model_dir = model_dir
+    return hparams
+
+
+def get_hparams_from_file(config_path):
+    with open(config_path, "r") as f:
+        data = f.read()
+    config = json.loads(data)
+
+    hparams = HParams(**config)
+    return hparams
+
+
+def check_git_hash(model_dir):
+    source_dir = os.path.dirname(os.path.realpath(__file__))
+    if not os.path.exists(os.path.join(source_dir, ".git")):
+        logger.warn('''{} is not a git repository, therefore hash value
+                       comparison will be ignored.'''.format(source_dir))
+        return
+
+    cur_hash = subprocess.getoutput("git rev-parse HEAD")
+
+    path = os.path.join(model_dir, "githash")
+    if os.path.exists(path):
+        saved_hash = open(path).read()
+        if saved_hash != cur_hash:
+            logger.warn(
+                "git hash values are different. {}(saved) != {}(current)".
+                format(saved_hash[:8], cur_hash[:8]))
+    else:
+        open(path, "w").write(cur_hash)
+
+
+def get_logger(model_dir, filename="train.log"):
+    global logger
+    logger = logging.getLogger(os.path.basename(model_dir))
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter(
+        "%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s")
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+    h = logging.FileHandler(os.path.join(model_dir, filename))
+    h.setLevel(logging.INFO)
+    h.setFormatter(formatter)
+    logger.addHandler(h)
+    return logger
 
 
 class HParams():
@@ -35,42 +305,3 @@ class HParams():
 
     def __repr__(self):
         return self.__dict__.__repr__()
-
-
-def load_checkpoint(checkpoint_path, model):
-    checkpoint_dict = load(checkpoint_path, map_location='cpu')
-    iteration = checkpoint_dict['iteration']
-    saved_state_dict = checkpoint_dict['model']
-    if hasattr(model, 'module'):
-        state_dict = model.module.state_dict()
-    else:
-        state_dict = model.state_dict()
-    new_state_dict = {}
-    for k, v in state_dict.items():
-        try:
-            new_state_dict[k] = saved_state_dict[k]
-        except:
-            logging.info("%s is not in the checkpoint" % k)
-            new_state_dict[k] = v
-        pass
-    if hasattr(model, 'module'):
-        model.module.load_state_dict(new_state_dict)
-    else:
-        model.load_state_dict(new_state_dict)
-    logging.info("Loaded checkpoint '{}' (iteration {})".format(
-        checkpoint_path, iteration))
-    return
-
-
-def get_hparams_from_file(config_path):
-    with open(config_path, "r") as f:
-        data = f.read()
-    config = loads(data)
-
-    hparams = HParams(**config)
-    return hparams
-
-
-def load_audio_to_torch(full_path, target_sampling_rate):
-    audio, sampling_rate = librosa.load(full_path, sr=target_sampling_rate, mono=True)
-    return FloatTensor(audio.astype(float32))