update readme

README.md

@@ -62,7 +62,7 @@ python preprocess.py config/finetune/preprocess.yaml
 Finetune speaker voice with
 
 ```
-python train.py [-h] [--pretrain_dir BASE_LINE_MODEL_PATH] [-p PREPROCESS_CONFIG_PATH] [-m MODEL_CONFIG_PATH] [-t TRAIN_CONFIG_PATH] [--vocoder_checkpoint VOCODER_CHECKPOINT_PATH] [--vocoder_config VOCODER_CONFIG_PATH]
+python finetune.py [-h] [--pretrain_dir BASE_LINE_MODEL_PATH] [-p PREPROCESS_CONFIG_PATH] [-m MODEL_CONFIG_PATH] [-t TRAIN_CONFIG_PATH] [--vocoder_checkpoint VOCODER_CHECKPOINT_PATH] [--vocoder_config VOCODER_CONFIG_PATH]
 ```
 
 # TensorBoard
@@ -72,6 +72,11 @@ Use
 ```
 tensorboard [--logdir LOG_PATH]
 ```
+* Tensorboard for pretrain model
+![](./assets/pretrain_tensorboard.png)
+
+* Tensorboard for finetune with only 5 sentences
+![](./assets/finetune_tensorboard.png)
 
 # References
 - [**AdaSpeech: Adaptive text to speech for custom voice**](https://arxiv.org/pdf/2103.00993.pdf).

config/pretrain/train.yaml

@@ -1,7 +1,7 @@
 path:
- ckpt_path: "./output/ckpt/Audiobook"
- log_path: "./output/log/Audiobook"
- result_path: "./output/result/Audiobook"
+ ckpt_path: "./output/ckpt/multi_language"
+ log_path: "./output/log/multi_language"
+ result_path: "./output/result/multi_language"
 optimizer:
  batch_size: 16
  betas: [0.9, 0.98]

inference.py

@@ -13,6 +13,7 @@
 from dataset import Dataset
 from text import text_to_sequence
 from datetime import datetime
+from g2p_en import G2p
 
 import audio as Audio
 
@@ -35,7 +36,7 @@ def get_vocoder(config, checkpoint_path):
 
 def synthesize(model, step, configs, vocoder, loader, control_values, output_dir):
  preprocess_config, model_config, train_config = configs
- pitch_control, energy_control, duration_control = control_values
+ pitch_control, energy_control, duration_control, eng_pos = control_values
  for batch in batchs:
  batch = to_device(batch, device)
  with torch.no_grad():
@@ -44,7 +45,7 @@ def synthesize(model, step, configs, vocoder, loader, control_values, output_dir
  *(batch[2:]),
  p_control=pitch_control,
  e_control=energy_control,
- d_control=duration_control
+ d_control=duration_control,
  )
  synth_samples(
  batch,
@@ -72,6 +73,13 @@ def get_reference_mel(reference_audio_dir, STFT):
  help="speaker ID for multi-speaker synthesis, for single-sentence mode only",
  )
 
+ parser.add_argument(
+ "--language_id",
+ type=int,
+ default=0,
+ help="language ID for multi-language synthesis"
+ )
+
  parser.add_argument(
  "--output_dir",
  type=str
@@ -106,10 +114,10 @@ def get_reference_mel(reference_audio_dir, STFT):
 
  # Read Config
  preprocess_config = yaml.load(
- open("config/1_CTV/preprocess.yaml", "r"), Loader=yaml.FullLoader
+ open("config/pretrain/preprocess.yaml", "r"), Loader=yaml.FullLoader
  )
- model_config = yaml.load(open("config/1_CTV/model.yaml", "r"), Loader=yaml.FullLoader)
- train_config = yaml.load(open("config/1_CTV/train.yaml", "r"), Loader=yaml.FullLoader)
+ model_config = yaml.load(open("config/pretrain/model.yaml", "r"), Loader=yaml.FullLoader)
+ train_config = yaml.load(open("config/pretrain/train.yaml", "r"), Loader=yaml.FullLoader)
  configs = (preprocess_config, model_config, train_config)
 
  output_dir = args.output_dir
@@ -131,10 +139,12 @@ def get_reference_mel(reference_audio_dir, STFT):
 
  # Preprocess texts
  ids = [datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p")]
- raw_texts = "Một người dân đang chạy thể dục dọc bờ sông Hương thì bất ngờ phát hiện quả đạn thời chiến tranh nằm bên bãi cỏ"
+ raw_texts = "thông tấn xã thailand cho rằng china đã quá tự cao tự đại trong mối quan hệ với russia"
  speakers = np.array([args.speaker_id])
- text = convert_text_to_ipa(raw_texts.lower())
+ languages = np.array([args.language_id])
+ text, eng_pos = convert_text_to_ipa(raw_texts.lower())
  text = text.replace(",", "sp")
+ print(text)
  text = np.array(
  text_to_sequence(
  text, preprocess_config["preprocessing"]["text"]["text_cleaners"]
@@ -144,8 +154,8 @@ def get_reference_mel(reference_audio_dir, STFT):
  text_lens = np.array([len(text[0])])
  mel_spectrogram = get_reference_mel(wav_path, STFT)
  mel_spectrogram = np.array([mel_spectrogram])
- batchs = [(ids, raw_texts, speakers, text, text_lens, max(text_lens), mel_spectrogram)]
+ batchs = [(ids, raw_texts, speakers, text, text_lens, max(text_lens), mel_spectrogram, languages)]
 
- control_values = args.pitch_control, args.energy_control, args.duration_control
+ control_values = args.pitch_control, args.energy_control, args.duration_control, eng_pos
 
  synthesize(model, args.restore_step, configs, vocoder, batchs, control_values, output_dir)