202 lines
6.3 KiB
YAML
202 lines
6.3 KiB
YAML
# set random seed, so that you may reproduce your result.
|
|
__set_seed1: !apply:random.seed [1986]
|
|
__set_seed2: !apply:numpy.random.seed [1986]
|
|
__set_seed3: !apply:torch.manual_seed [1986]
|
|
__set_seed4: !apply:torch.cuda.manual_seed_all [1986]
|
|
|
|
# fixed params
|
|
sample_rate: 22050
|
|
text_encoder_input_size: 512
|
|
llm_input_size: 1024
|
|
llm_output_size: 1024
|
|
spk_embed_dim: 192
|
|
|
|
# model params
|
|
# for all class/function included in this repo, we use !<name> or !<new> for intialization, so that user may find all corresponding class/function according to one single yaml.
|
|
# for system/third_party class/function, we do not require this.
|
|
llm: !new:cosyvoice.llm.llm.TransformerLM
|
|
text_encoder_input_size: !ref <text_encoder_input_size>
|
|
llm_input_size: !ref <llm_input_size>
|
|
llm_output_size: !ref <llm_output_size>
|
|
text_token_size: 51866
|
|
speech_token_size: 4096
|
|
length_normalized_loss: True
|
|
lsm_weight: 0
|
|
spk_embed_dim: !ref <spk_embed_dim>
|
|
text_encoder: !new:cosyvoice.transformer.encoder.ConformerEncoder
|
|
input_size: !ref <text_encoder_input_size>
|
|
output_size: 1024
|
|
attention_heads: 16
|
|
linear_units: 4096
|
|
num_blocks: 6
|
|
dropout_rate: 0.1
|
|
positional_dropout_rate: 0.1
|
|
attention_dropout_rate: 0.0
|
|
normalize_before: True
|
|
input_layer: 'linear'
|
|
pos_enc_layer_type: 'rel_pos_espnet'
|
|
selfattention_layer_type: 'rel_selfattn'
|
|
use_cnn_module: False
|
|
macaron_style: False
|
|
use_dynamic_chunk: False
|
|
use_dynamic_left_chunk: False
|
|
static_chunk_size: 1
|
|
llm: !new:cosyvoice.transformer.encoder.TransformerEncoder
|
|
input_size: !ref <llm_input_size>
|
|
output_size: !ref <llm_output_size>
|
|
attention_heads: 16
|
|
linear_units: 4096
|
|
num_blocks: 14
|
|
dropout_rate: 0.1
|
|
positional_dropout_rate: 0.1
|
|
attention_dropout_rate: 0.0
|
|
input_layer: 'linear_legacy'
|
|
pos_enc_layer_type: 'rel_pos_espnet'
|
|
selfattention_layer_type: 'rel_selfattn'
|
|
static_chunk_size: 1
|
|
sampling: !name:cosyvoice.utils.common.ras_sampling
|
|
top_p: 0.8
|
|
top_k: 25
|
|
win_size: 10
|
|
tau_r: 0.1
|
|
|
|
flow: !new:cosyvoice.flow.flow.MaskedDiffWithXvec
|
|
input_size: 512
|
|
output_size: 80
|
|
spk_embed_dim: !ref <spk_embed_dim>
|
|
output_type: 'mel'
|
|
vocab_size: 4096
|
|
input_frame_rate: 50
|
|
only_mask_loss: True
|
|
encoder: !new:cosyvoice.transformer.encoder.ConformerEncoder
|
|
output_size: 512
|
|
attention_heads: 8
|
|
linear_units: 2048
|
|
num_blocks: 6
|
|
dropout_rate: 0.1
|
|
positional_dropout_rate: 0.1
|
|
attention_dropout_rate: 0.1
|
|
normalize_before: True
|
|
input_layer: 'linear'
|
|
pos_enc_layer_type: 'rel_pos_espnet'
|
|
selfattention_layer_type: 'rel_selfattn'
|
|
input_size: 512
|
|
use_cnn_module: False
|
|
macaron_style: False
|
|
length_regulator: !new:cosyvoice.flow.length_regulator.InterpolateRegulator
|
|
channels: 80
|
|
sampling_ratios: [1, 1, 1, 1]
|
|
decoder: !new:cosyvoice.flow.flow_matching.ConditionalCFM
|
|
in_channels: 240
|
|
n_spks: 1
|
|
spk_emb_dim: 80
|
|
cfm_params: !new:omegaconf.DictConfig
|
|
content:
|
|
sigma_min: 1e-06
|
|
solver: 'euler'
|
|
t_scheduler: 'cosine'
|
|
training_cfg_rate: 0.2
|
|
inference_cfg_rate: 0.7
|
|
reg_loss_type: 'l1'
|
|
estimator: !new:cosyvoice.flow.decoder.ConditionalDecoder
|
|
in_channels: 320
|
|
out_channels: 80
|
|
channels: [256, 256]
|
|
dropout: 0.0
|
|
attention_head_dim: 64
|
|
n_blocks: 4
|
|
num_mid_blocks: 12
|
|
num_heads: 8
|
|
act_fn: 'gelu'
|
|
|
|
hift: !new:cosyvoice.hifigan.generator.HiFTGenerator
|
|
in_channels: 80
|
|
base_channels: 512
|
|
nb_harmonics: 8
|
|
sampling_rate: !ref <sample_rate>
|
|
nsf_alpha: 0.1
|
|
nsf_sigma: 0.003
|
|
nsf_voiced_threshold: 10
|
|
upsample_rates: [8, 8]
|
|
upsample_kernel_sizes: [16, 16]
|
|
istft_params:
|
|
n_fft: 16
|
|
hop_len: 4
|
|
resblock_kernel_sizes: [3, 7, 11]
|
|
resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
|
|
source_resblock_kernel_sizes: [7, 11]
|
|
source_resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5]]
|
|
lrelu_slope: 0.1
|
|
audio_limit: 0.99
|
|
f0_predictor: !new:cosyvoice.hifigan.f0_predictor.ConvRNNF0Predictor
|
|
num_class: 1
|
|
in_channels: 80
|
|
cond_channels: 512
|
|
|
|
# processor functions
|
|
parquet_opener: !name:cosyvoice.dataset.processor.parquet_opener
|
|
get_tokenizer: !name:whisper.tokenizer.get_tokenizer
|
|
multilingual: True
|
|
num_languages: 100
|
|
language: 'en'
|
|
task: 'transcribe'
|
|
allowed_special: 'all'
|
|
tokenize: !name:cosyvoice.dataset.processor.tokenize
|
|
get_tokenizer: !ref <get_tokenizer>
|
|
allowed_special: !ref <allowed_special>
|
|
filter: !name:cosyvoice.dataset.processor.filter
|
|
max_length: 40960
|
|
min_length: 0
|
|
token_max_length: 200
|
|
token_min_length: 1
|
|
resample: !name:cosyvoice.dataset.processor.resample
|
|
resample_rate: !ref <sample_rate>
|
|
feat_extractor: !name:matcha.utils.audio.mel_spectrogram
|
|
n_fft: 1024
|
|
num_mels: 80
|
|
sampling_rate: !ref <sample_rate>
|
|
hop_size: 256
|
|
win_size: 1024
|
|
fmin: 0
|
|
fmax: 8000
|
|
center: False
|
|
compute_fbank: !name:cosyvoice.dataset.processor.compute_fbank
|
|
feat_extractor: !ref <feat_extractor>
|
|
parse_embedding: !name:cosyvoice.dataset.processor.parse_embedding
|
|
normalize: True
|
|
shuffle: !name:cosyvoice.dataset.processor.shuffle
|
|
shuffle_size: 1000
|
|
sort: !name:cosyvoice.dataset.processor.sort
|
|
sort_size: 500 # sort_size should be less than shuffle_size
|
|
batch: !name:cosyvoice.dataset.processor.batch
|
|
batch_type: 'dynamic'
|
|
max_frames_in_batch: 2000
|
|
padding: !name:cosyvoice.dataset.processor.padding
|
|
|
|
# dataset processor pipeline
|
|
data_pipeline: [
|
|
!ref <parquet_opener>,
|
|
!ref <tokenize>,
|
|
!ref <filter>,
|
|
!ref <resample>,
|
|
!ref <compute_fbank>,
|
|
!ref <parse_embedding>,
|
|
!ref <shuffle>,
|
|
!ref <sort>,
|
|
!ref <batch>,
|
|
!ref <padding>,
|
|
]
|
|
|
|
# train conf
|
|
train_conf:
|
|
optim: adam
|
|
optim_conf:
|
|
lr: 0.001
|
|
scheduler: warmuplr
|
|
scheduler_conf:
|
|
warmup_steps: 2500
|
|
max_epoch: 200
|
|
grad_clip: 5
|
|
accum_grad: 2
|
|
log_interval: 100
|
|
save_per_step: -1 |