asr_oxford_frontend_raw_cy_bpe150_sp/train.1.log

# python3 -m espnet2.bin.asr_train --use_preprocessor true --bpemodel data/cy_token_list/bpe_unigram150/bpe.model --token_type bpe --token_list data/cy_token_list/bpe_unigram150/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_cy/wav.scp,speech,sound --valid_data_path_and_name_and_type dump/raw/dev_cy/text,text,text --valid_shape_file exp/asr_stats_raw_cy_bpe150_sp/valid/speech_shape --valid_shape_file exp/asr_stats_raw_cy_bpe150_sp/valid/text_shape.bpe --resume true --init_param --ignore_init_mismatch false --fold_length 80000 --fold_length 150 --output_dir exp/asr_oxford_frontend_raw_cy_bpe150_sp --config conf/tuning/oxford_frontend.yaml --frontend_conf fs=16k --train_data_path_and_name_and_type dump/raw/train_cy_sp/wav.scp,speech,sound --train_data_path_and_name_and_type dump/raw/train_cy_sp/text,text,text --train_shape_file exp/asr_stats_raw_cy_bpe150_sp/train/speech_shape --train_shape_file exp/asr_stats_raw_cy_bpe150_sp/train/text_shape.bpe --ngpu 3 --multiprocessing_distributed True 
# Started at Mon Jun  6 13:46:59 EDT 2022
#
/usr/bin/python3 /project/ocean/junweih/espnet/espnet2/bin/asr_train.py --use_preprocessor true --bpemodel data/cy_token_list/bpe_unigram150/bpe.model --token_type bpe --token_list data/cy_token_list/bpe_unigram150/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_cy/wav.scp,speech,sound --valid_data_path_and_name_and_type dump/raw/dev_cy/text,text,text --valid_shape_file exp/asr_stats_raw_cy_bpe150_sp/valid/speech_shape --valid_shape_file exp/asr_stats_raw_cy_bpe150_sp/valid/text_shape.bpe --resume true --init_param --ignore_init_mismatch false --fold_length 80000 --fold_length 150 --output_dir exp/asr_oxford_frontend_raw_cy_bpe150_sp --config conf/tuning/oxford_frontend.yaml --frontend_conf fs=16k --train_data_path_and_name_and_type dump/raw/train_cy_sp/wav.scp,speech,sound --train_data_path_and_name_and_type dump/raw/train_cy_sp/text,text,text --train_shape_file exp/asr_stats_raw_cy_bpe150_sp/train/speech_shape --train_shape_file exp/asr_stats_raw_cy_bpe150_sp/train/text_shape.bpe --ngpu 3 --multiprocessing_distributed True
[islpc50:0/3] 2022-06-06 13:47:13,287 (distributed_c10d:217) INFO: Added key: store_based_barrier_key:1 to store for rank: 0
[islpc50:0/3] 2022-06-06 13:47:13,288 (distributed_c10d:251) INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 3 nodes.
[islpc50:0/3] 2022-06-06 13:47:13,348 (asr:411) INFO: Vocabulary size: 150
[islpc50:0/3] 2022-06-06 13:47:13,888 (filelock:274) INFO: Lock 139997783706976 acquired on ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f.lock
[islpc50:0/3] 2022-06-06 13:47:13,889 (filelock:318) INFO: Lock 139997783706976 released on ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f.lock
[Featurizer] - The selected feature last_hidden_state's downsample rate is 320
[islpc50:0/3] 2022-06-06 13:47:25,154 (s3prl:159) INFO: Pretrained S3PRL frontend model parameters reloaded!
[islpc50:0/3] 2022-06-06 13:47:29,051 (abs_task:1157) INFO: pytorch.version=1.10.1+cu111, cuda.available=True, cudnn.version=8005, cudnn.benchmark=False, cudnn.deterministic=True
[islpc50:0/3] 2022-06-06 13:47:29,056 (abs_task:1158) INFO: Model structure:
ESPnetASRModel(
  (frontend): S3prlFrontend(
    (upstream): UpstreamExpert(
      (model): Wav2Vec2Model(
        (feature_extractor): ConvFeatureExtractionModel(
          (conv_layers): ModuleList(
            (0): Sequential(
              (0): Conv1d(1, 512, kernel_size=(10,), stride=(5,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (1): Sequential(
              (0): Conv1d(512, 512, kernel_size=(3,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (2): Sequential(
              (0): Conv1d(512, 512, kernel_size=(3,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (3): Sequential(
              (0): Conv1d(512, 512, kernel_size=(3,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (4): Sequential(
              (0): Conv1d(512, 512, kernel_size=(3,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (5): Sequential(
              (0): Conv1d(512, 512, kernel_size=(2,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (6): Sequential(
              (0): Conv1d(512, 512, kernel_size=(2,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
          )
        )
        (post_extract_proj): Linear(in_features=512, out_features=1024, bias=True)
        (dropout_input): Dropout(p=0.1, inplace=False)
        (dropout_features): Dropout(p=0.1, inplace=False)
        (quantizer): GumbelVectorQuantizer(
          (weight_proj): Linear(in_features=512, out_features=640, bias=True)
        )
        (project_q): Linear(in_features=768, out_features=768, bias=True)
        (encoder): TransformerEncoder(
          (pos_conv): Sequential(
            (0): Conv1d(1024, 1024, kernel_size=(128,), stride=(1,), padding=(64,), groups=16)
            (1): SamePad()
            (2): GELU()
          )
          (layers): ModuleList(
            (0): AdapterTransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (adapter1): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
              (adapter2): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
            )
            (1): AdapterTransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (adapter1): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
              (adapter2): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
            )
            (2): AdapterTransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (adapter1): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
              (adapter2): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
            )
            (3): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (4): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (5): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (6): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (7): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (8): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (9): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (10): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (11): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (12): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (13): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (14): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (15): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (16): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (17): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (18): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (19): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (20): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (21): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (22): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (23): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
          )
          (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
        )
        (layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
        (final_proj): Linear(in_features=1024, out_features=768, bias=True)
      )
    )
    (featurizer): Featurizer()
  )
  (normalize): UtteranceMVN(norm_means=True, norm_vars=False)
  (encoder): RNNEncoder(
    (enc): ModuleList(
      (0): RNNP(
        (birnn0): LSTM(1024, 320, batch_first=True, bidirectional=True)
        (bt0): Linear(in_features=640, out_features=320, bias=True)
        (birnn1): LSTM(320, 320, batch_first=True, bidirectional=True)
        (bt1): Linear(in_features=640, out_features=320, bias=True)
        (birnn2): LSTM(320, 320, batch_first=True, bidirectional=True)
        (bt2): Linear(in_features=640, out_features=320, bias=True)
        (birnn3): LSTM(320, 320, batch_first=True, bidirectional=True)
        (bt3): Linear(in_features=640, out_features=320, bias=True)
      )
    )
  )
  (criterion_att): LabelSmoothingLoss(
    (criterion): KLDivLoss()
  )
  (ctc): CTC(
    (ctc_lo): Linear(in_features=320, out_features=150, bias=True)
    (ctc_loss): CTCLoss()
  )
)

Model summary:
    Class Name: ESPnetASRModel
    Total Number of model parameters: 329.00 M
    Number of trainable parameters: 11.61 M (3.5%)
    Size: 46.45 MB
    Type: torch.float32
[islpc50:0/3] 2022-06-06 13:47:29,056 (abs_task:1161) INFO: Optimizer:
Adam (
Parameter Group 0
    amsgrad: False
    betas: (0.9, 0.999)
    eps: 1e-08
    initial_lr: 0.00027
    lr: 6.749999999999999e-09
    weight_decay: 0
)
[islpc50:0/3] 2022-06-06 13:47:29,057 (abs_task:1162) INFO: Scheduler: WarmupLR(warmup_steps=40000)
[islpc50:0/3] 2022-06-06 13:47:29,069 (abs_task:1171) INFO: Saving the configuration in exp/asr_oxford_frontend_raw_cy_bpe150_sp/config.yaml
[islpc50:0/3] 2022-06-06 13:47:31,251 (abs_task:1525) INFO: [train] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/train_cy_sp/wav.scp", "type": "sound"}
  text: {"path": "dump/raw/train_cy_sp/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.CommonPreprocessor object at 0x7f5305f45100>)
[islpc50:0/3] 2022-06-06 13:47:31,251 (abs_task:1526) INFO: [train] Batch sampler: FoldedBatchSampler(N-batch=24097, batch_size=10, shape_files=['exp/asr_stats_raw_cy_bpe150_sp/train/speech_shape', 'exp/asr_stats_raw_cy_bpe150_sp/train/text_shape.bpe'], sort_in_batch=descending, sort_batch=descending)
[islpc50:0/3] 2022-06-06 13:47:31,254 (abs_task:1527) INFO: [train] mini-batch sizes summary: N-batch=24097, mean=7.3, min=3, max=10
[islpc50:0/3] 2022-06-06 13:47:31,343 (abs_task:1525) INFO: [valid] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/dev_cy/wav.scp", "type": "sound"}
  text: {"path": "dump/raw/dev_cy/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.CommonPreprocessor object at 0x7f5306198760>)
[islpc50:0/3] 2022-06-06 13:47:31,343 (abs_task:1526) INFO: [valid] Batch sampler: FoldedBatchSampler(N-batch=464, batch_size=10, shape_files=['exp/asr_stats_raw_cy_bpe150_sp/valid/speech_shape', 'exp/asr_stats_raw_cy_bpe150_sp/valid/text_shape.bpe'], sort_in_batch=descending, sort_batch=descending)
[islpc50:0/3] 2022-06-06 13:47:31,343 (abs_task:1527) INFO: [valid] mini-batch sizes summary: N-batch=464, mean=6.3, min=3, max=10
[islpc50:0/3] 2022-06-06 13:47:31,380 (abs_task:1525) INFO: [plot_att] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/dev_cy/wav.scp", "type": "sound"}
  text: {"path": "dump/raw/dev_cy/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.CommonPreprocessor object at 0x7f530604b580>)
[islpc50:0/3] 2022-06-06 13:47:31,380 (abs_task:1526) INFO: [plot_att] Batch sampler: UnsortedBatchSampler(N-batch=2933, batch_size=1, key_file=exp/asr_stats_raw_cy_bpe150_sp/valid/speech_shape, 
[islpc50:0/3] 2022-06-06 13:47:31,380 (abs_task:1527) INFO: [plot_att] mini-batch sizes summary: N-batch=3, mean=1.0, min=1, max=1
islpc50:1026619:1026619 [0] NCCL INFO Bootstrap : Using bond0:128.2.205.9<0>
islpc50:1026619:1026619 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
islpc50:1026619:1026619 [0] NCCL INFO NET/IB : No device found.
islpc50:1026619:1026619 [0] NCCL INFO NET/Socket : Using [0]bond0:128.2.205.9<0>
islpc50:1026619:1026619 [0] NCCL INFO Using network Socket
NCCL version 2.10.3+cuda11.1
islpc50:1026620:1026620 [1] NCCL INFO Bootstrap : Using bond0:128.2.205.9<0>
islpc50:1026621:1026621 [2] NCCL INFO Bootstrap : Using bond0:128.2.205.9<0>
islpc50:1026620:1026620 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
islpc50:1026621:1026621 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
islpc50:1026620:1026620 [1] NCCL INFO NET/IB : No device found.
islpc50:1026621:1026621 [2] NCCL INFO NET/IB : No device found.
islpc50:1026620:1026620 [1] NCCL INFO NET/Socket : Using [0]bond0:128.2.205.9<0>
islpc50:1026620:1026620 [1] NCCL INFO Using network Socket
islpc50:1026621:1026621 [2] NCCL INFO NET/Socket : Using [0]bond0:128.2.205.9<0>
islpc50:1026621:1026621 [2] NCCL INFO Using network Socket
islpc50:1026619:1026741 [0] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026619:1026741 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026619:1026741 [0] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026619:1026741 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026620:1026742 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026620:1026742 [1] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026620:1026742 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026620:1026742 [1] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026621:1026743 [2] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026621:1026743 [2] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026621:1026743 [2] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026621:1026743 [2] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026621:1026743 [2] NCCL INFO Trees [0] -1/-1/-1->2->1 [1] -1/-1/-1->2->1
islpc50:1026620:1026742 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0
islpc50:1026621:1026743 [2] NCCL INFO Setting affinity for GPU 2 to ffffff
islpc50:1026619:1026741 [0] NCCL INFO Channel 00/02 :    0   1   2
islpc50:1026620:1026742 [1] NCCL INFO Setting affinity for GPU 1 to ffffff
islpc50:1026619:1026741 [0] NCCL INFO Channel 01/02 :    0   1   2
islpc50:1026619:1026741 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1
islpc50:1026619:1026741 [0] NCCL INFO Setting affinity for GPU 0 to ffffff
islpc50:1026620:1026742 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026620:1026742 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026619:1026741 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026621:1026743 [2] NCCL INFO Channel 00 : 2[67000] -> 0[19000] via direct shared memory
islpc50:1026620:1026742 [1] NCCL INFO Channel 00 : 1[1a000] -> 2[67000] via direct shared memory
islpc50:1026621:1026743 [2] NCCL INFO Channel 01 : 2[67000] -> 0[19000] via direct shared memory
islpc50:1026619:1026741 [0] NCCL INFO Channel 00 : 0[19000] -> 1[1a000] via direct shared memory
islpc50:1026620:1026742 [1] NCCL INFO Channel 01 : 1[1a000] -> 2[67000] via direct shared memory
islpc50:1026619:1026741 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026619:1026741 [0] NCCL INFO Channel 01 : 0[19000] -> 1[1a000] via direct shared memory
islpc50:1026621:1026743 [2] NCCL INFO Connected all rings
islpc50:1026620:1026742 [1] NCCL INFO Connected all rings
islpc50:1026619:1026741 [0] NCCL INFO Connected all rings
islpc50:1026621:1026743 [2] NCCL INFO Channel 00 : 2[67000] -> 1[1a000] via direct shared memory
islpc50:1026619:1026741 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026621:1026743 [2] NCCL INFO Channel 01 : 2[67000] -> 1[1a000] via direct shared memory
islpc50:1026619:1026741 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1026620:1026742 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026620:1026742 [1] NCCL INFO Channel 00 : 1[1a000] -> 0[19000] via direct shared memory
islpc50:1026620:1026742 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1026620:1026742 [1] NCCL INFO Channel 01 : 1[1a000] -> 0[19000] via direct shared memory
islpc50:1026619:1026741 [0] NCCL INFO Connected all trees
islpc50:1026619:1026741 [0] NCCL INFO threadThresholds 8/8/64 | 24/8/64 | 8/8/512
islpc50:1026619:1026741 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
islpc50:1026620:1026742 [1] NCCL INFO Connected all trees
islpc50:1026620:1026742 [1] NCCL INFO threadThresholds 8/8/64 | 24/8/64 | 8/8/512
islpc50:1026620:1026742 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
islpc50:1026621:1026743 [2] NCCL INFO Connected all trees
islpc50:1026621:1026743 [2] NCCL INFO threadThresholds 8/8/64 | 24/8/64 | 8/8/512
islpc50:1026621:1026743 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
islpc50:1026619:1026741 [0] NCCL INFO comm 0x7f52b4002fb0 rank 0 nranks 3 cudaDev 0 busId 19000 - Init COMPLETE
islpc50:1026620:1026742 [1] NCCL INFO comm 0x7f519c002fb0 rank 1 nranks 3 cudaDev 1 busId 1a000 - Init COMPLETE
islpc50:1026619:1026619 [0] NCCL INFO Launch mode Parallel
islpc50:1026621:1026743 [2] NCCL INFO comm 0x7f76c0002fb0 rank 2 nranks 3 cudaDev 2 busId 67000 - Init COMPLETE
[islpc50:0/3] 2022-06-06 13:47:31,916 (trainer:280) INFO: 1/18epoch started
[s3prl.upstream.experts] Warning: can not import s3prl.upstream.byol_a.expert: No module named 'easydict'. Pass.
[s3prl.hub] Warning: can not import s3prl.upstream.byol_a.hubconf: No module named 'easydict'. Please see upstream/byol_a/README.md
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.quesst14_dtw.expert: No module named 'dtw'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.separation_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.enhancement_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.speech_commands.expert: No module named 'catalyst'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.a2a-vc-vctk.expert: No module named 'resemblyzer'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.voxceleb2_ge2e.expert: No module named 'sox'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.sv_voxceleb1.expert: No module named 'sox'. Pass.
Using cache found in ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f
for https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_new.pt
>> inserted adapters to the following layers: 0, 1, 2
  * original model weights: 317,390,592
  * new model weights - all: 319,757,184
  * new model weights - trainable: 2,366,592 ( 0.75% of original model)
[s3prl.upstream.experts] Warning: can not import s3prl.upstream.byol_a.expert: No module named 'easydict'. Pass.
[s3prl.hub] Warning: can not import s3prl.upstream.byol_a.hubconf: No module named 'easydict'. Please see upstream/byol_a/README.md
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.quesst14_dtw.expert: No module named 'dtw'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.separation_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.enhancement_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.speech_commands.expert: No module named 'catalyst'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.a2a-vc-vctk.expert: No module named 'resemblyzer'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.voxceleb2_ge2e.expert: No module named 'sox'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.sv_voxceleb1.expert: No module named 'sox'. Pass.
Using cache found in ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f
for https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_new.pt
>> inserted adapters to the following layers: 0, 1, 2
  * original model weights: 317,390,592
  * new model weights - all: 319,757,184
  * new model weights - trainable: 2,366,592 ( 0.75% of original model)
[s3prl.upstream.experts] Warning: can not import s3prl.upstream.byol_a.expert: No module named 'easydict'. Pass.
[s3prl.hub] Warning: can not import s3prl.upstream.byol_a.hubconf: No module named 'easydict'. Please see upstream/byol_a/README.md
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.quesst14_dtw.expert: No module named 'dtw'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.separation_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.enhancement_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.speech_commands.expert: No module named 'catalyst'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.a2a-vc-vctk.expert: No module named 'resemblyzer'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.voxceleb2_ge2e.expert: No module named 'sox'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.sv_voxceleb1.expert: No module named 'sox'. Pass.
Using cache found in ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f
for https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_new.pt
>> inserted adapters to the following layers: 0, 1, 2
  * original model weights: 317,390,592
  * new model weights - all: 319,757,184
  * new model weights - trainable: 2,366,592 ( 0.75% of original model)
[islpc50:0/3] 2022-06-06 13:47:38,154 (distributed:874) INFO: Reducer buckets have been rebuilt in this iteration.
[islpc50:0/3] 2022-06-06 13:51:56,624 (trainer:678) INFO: 1epoch:train:1-1204batch: iter_time=2.466e-04, forward_time=0.086, loss_ctc=171.000, loss=171.000, backward_time=0.052, optim_step_time=0.004, optim0_lr0=4.074e-06, train_time=0.220
[islpc50:0/3] 2022-06-06 13:56:17,520 (trainer:678) INFO: 1epoch:train:1205-2408batch: iter_time=8.446e-05, forward_time=0.087, loss_ctc=116.693, loss=116.693, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.220e-05, train_time=0.217
[islpc50:0/3] 2022-06-06 14:00:39,769 (trainer:678) INFO: 1epoch:train:2409-3612batch: iter_time=8.629e-05, forward_time=0.087, loss_ctc=115.924, loss=115.924, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.033e-05, train_time=0.218
[islpc50:0/3] 2022-06-06 14:05:01,060 (trainer:678) INFO: 1epoch:train:3613-4816batch: iter_time=8.032e-05, forward_time=0.087, loss_ctc=114.083, loss=114.083, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.845e-05, train_time=0.217
[islpc50:0/3] 2022-06-06 14:09:22,286 (trainer:678) INFO: 1epoch:train:4817-6020batch: iter_time=8.354e-05, forward_time=0.087, loss_ctc=112.166, loss=112.166, backward_time=0.052, optim_step_time=0.004, optim0_lr0=3.658e-05, train_time=0.217
[islpc50:0/3] 2022-06-06 14:13:42,685 (trainer:678) INFO: 1epoch:train:6021-7224batch: iter_time=8.214e-05, forward_time=0.087, loss_ctc=111.017, loss=111.017, backward_time=0.051, optim_step_time=0.004, optim0_lr0=4.471e-05, train_time=0.216
[islpc50:0/3] 2022-06-06 14:18:01,963 (trainer:678) INFO: 1epoch:train:7225-8428batch: iter_time=8.061e-05, forward_time=0.086, loss_ctc=106.179, loss=106.179, backward_time=0.052, optim_step_time=0.004, optim0_lr0=5.284e-05, train_time=0.215
[islpc50:0/3] 2022-06-06 14:22:22,125 (trainer:678) INFO: 1epoch:train:8429-9632batch: iter_time=8.164e-05, forward_time=0.087, loss_ctc=105.491, loss=105.491, backward_time=0.052, optim_step_time=0.004, optim0_lr0=6.096e-05, train_time=0.216
[islpc50:0/3] 2022-06-06 14:26:43,267 (trainer:678) INFO: 1epoch:train:9633-10836batch: iter_time=8.613e-05, forward_time=0.087, loss_ctc=100.576, loss=100.576, backward_time=0.052, optim_step_time=0.004, optim0_lr0=6.909e-05, train_time=0.217
[islpc50:0/3] 2022-06-06 14:31:02,378 (trainer:678) INFO: 1epoch:train:10837-12040batch: iter_time=8.333e-05, forward_time=0.086, loss_ctc=91.261, loss=91.261, backward_time=0.052, optim_step_time=0.004, optim0_lr0=7.722e-05, train_time=0.215
[islpc50:0/3] 2022-06-06 14:35:21,133 (trainer:678) INFO: 1epoch:train:12041-13244batch: iter_time=8.830e-05, forward_time=0.086, loss_ctc=83.260, loss=83.260, backward_time=0.052, optim_step_time=0.004, optim0_lr0=8.534e-05, train_time=0.215
[islpc50:0/3] 2022-06-06 14:39:41,591 (trainer:678) INFO: 1epoch:train:13245-14448batch: iter_time=7.230e-04, forward_time=0.087, loss_ctc=74.298, loss=74.298, backward_time=0.051, optim_step_time=0.004, optim0_lr0=9.347e-05, train_time=0.216
[islpc50:0/3] 2022-06-06 14:43:59,478 (trainer:678) INFO: 1epoch:train:14449-15652batch: iter_time=8.202e-05, forward_time=0.086, loss_ctc=65.073, loss=65.073, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.016e-04, train_time=0.214
[islpc50:0/3] 2022-06-06 14:48:22,207 (trainer:678) INFO: 1epoch:train:15653-16856batch: iter_time=8.325e-05, forward_time=0.088, loss_ctc=59.709, loss=59.709, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.097e-04, train_time=0.218
[islpc50:0/3] 2022-06-06 14:52:43,361 (trainer:678) INFO: 1epoch:train:16857-18060batch: iter_time=8.362e-05, forward_time=0.087, loss_ctc=52.398, loss=52.398, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.179e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 14:57:02,868 (trainer:678) INFO: 1epoch:train:18061-19264batch: iter_time=8.149e-05, forward_time=0.086, loss_ctc=46.697, loss=46.697, backward_time=0.051, optim_step_time=0.004, optim0_lr0=1.260e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 15:01:23,155 (trainer:678) INFO: 1epoch:train:19265-20468batch: iter_time=8.251e-05, forward_time=0.086, loss_ctc=43.592, loss=43.592, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.341e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 15:05:42,696 (trainer:678) INFO: 1epoch:train:20469-21672batch: iter_time=8.756e-05, forward_time=0.086, loss_ctc=39.917, loss=39.917, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.422e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 15:10:04,627 (trainer:678) INFO: 1epoch:train:21673-22876batch: iter_time=8.321e-05, forward_time=0.087, loss_ctc=37.551, loss=37.551, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.504e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 15:14:24,272 (trainer:678) INFO: 1epoch:train:22877-24080batch: iter_time=8.564e-05, forward_time=0.086, loss_ctc=34.346, loss=34.346, backward_time=0.051, optim_step_time=0.004, optim0_lr0=1.585e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 15:15:54,079 (trainer:334) INFO: 1epoch results: [train] iter_time=1.238e-04, forward_time=0.087, loss_ctc=83.908, loss=83.908, backward_time=0.052, optim_step_time=0.004, optim0_lr0=8.134e-05, train_time=0.216, time=1 hour, 26 minutes and 57.31 seconds, total_count=24097, gpu_max_cached_mem_GB=5.582, [valid] loss_ctc=56.700, cer_ctc=0.315, loss_att=nan, acc=nan, cer=nan, wer=nan, loss=56.700, time=1 minute and 17.1 seconds, total_count=464, gpu_max_cached_mem_GB=5.582, [att_plot] time=7.71 seconds, total_count=0, gpu_max_cached_mem_GB=5.582
[islpc50:0/3] 2022-06-06 15:16:32,908 (trainer:382) INFO: The best model has been updated: train.loss, valid.loss, valid.acc
[islpc50:0/3] 2022-06-06 15:16:32,909 (trainer:268) INFO: 2/18epoch started. Estimated time to finish: 1 day, 1 hour and 13 minutes
[islpc50:0/3] 2022-06-06 15:20:59,560 (trainer:678) INFO: 2epoch:train:1-1204batch: iter_time=2.212e-04, forward_time=0.086, loss_ctc=31.180, loss=31.180, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.667e-04, train_time=0.221
[islpc50:0/3] 2022-06-06 15:25:21,985 (trainer:678) INFO: 2epoch:train:1205-2408batch: iter_time=8.865e-05, forward_time=0.087, loss_ctc=28.961, loss=28.961, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.749e-04, train_time=0.218
[islpc50:0/3] 2022-06-06 15:29:43,025 (trainer:678) INFO: 2epoch:train:2409-3612batch: iter_time=8.477e-05, forward_time=0.086, loss_ctc=27.531, loss=27.531, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.830e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 15:34:03,991 (trainer:678) INFO: 2epoch:train:3613-4816batch: iter_time=8.628e-05, forward_time=0.086, loss_ctc=26.807, loss=26.807, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.911e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 15:38:22,628 (trainer:678) INFO: 2epoch:train:4817-6020batch: iter_time=8.610e-05, forward_time=0.085, loss_ctc=23.131, loss=23.131, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.992e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 15:42:44,407 (trainer:678) INFO: 2epoch:train:6021-7224batch: iter_time=8.574e-05, forward_time=0.087, loss_ctc=22.992, loss=22.992, backward_time=0.054, optim_step_time=0.004, optim0_lr0=2.074e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 15:47:04,748 (trainer:678) INFO: 2epoch:train:7225-8428batch: iter_time=8.906e-05, forward_time=0.086, loss_ctc=21.908, loss=21.908, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.155e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 15:51:24,528 (trainer:678) INFO: 2epoch:train:8429-9632batch: iter_time=8.260e-05, forward_time=0.086, loss_ctc=20.819, loss=20.819, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.236e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 15:55:45,202 (trainer:678) INFO: 2epoch:train:9633-10836batch: iter_time=8.235e-05, forward_time=0.086, loss_ctc=20.234, loss=20.234, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.317e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 16:00:06,173 (trainer:678) INFO: 2epoch:train:10837-12040batch: iter_time=8.796e-05, forward_time=0.087, loss_ctc=19.452, loss=19.452, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.399e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 16:04:30,700 (trainer:678) INFO: 2epoch:train:12041-13244batch: iter_time=8.349e-05, forward_time=0.088, loss_ctc=18.823, loss=18.823, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.480e-04, train_time=0.220
[islpc50:0/3] 2022-06-06 16:08:52,164 (trainer:678) INFO: 2epoch:train:13245-14448batch: iter_time=8.407e-05, forward_time=0.086, loss_ctc=19.243, loss=19.243, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.561e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 16:13:11,686 (trainer:678) INFO: 2epoch:train:14449-15652batch: iter_time=2.488e-04, forward_time=0.086, loss_ctc=17.942, loss=17.942, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.643e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 16:17:31,732 (trainer:678) INFO: 2epoch:train:15653-16856batch: iter_time=8.266e-05, forward_time=0.086, loss_ctc=17.304, loss=17.304, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.686e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 16:21:52,524 (trainer:678) INFO: 2epoch:train:16857-18060batch: iter_time=8.752e-05, forward_time=0.086, loss_ctc=16.702, loss=16.702, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.649e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 16:26:13,409 (trainer:678) INFO: 2epoch:train:18061-19264batch: iter_time=8.778e-05, forward_time=0.086, loss_ctc=16.521, loss=16.521, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.611e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 16:30:33,489 (trainer:678) INFO: 2epoch:train:19265-20468batch: iter_time=8.700e-05, forward_time=0.086, loss_ctc=15.392, loss=15.392, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.575e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 16:34:53,439 (trainer:678) INFO: 2epoch:train:20469-21672batch: iter_time=8.891e-05, forward_time=0.086, loss_ctc=15.119, loss=15.119, backward_time=0.051, optim_step_time=0.004, optim0_lr0=2.541e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 16:39:13,420 (trainer:678) INFO: 2epoch:train:21673-22876batch: iter_time=8.356e-05, forward_time=0.086, loss_ctc=14.312, loss=14.312, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.508e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 16:43:30,938 (trainer:678) INFO: 2epoch:train:22877-24080batch: iter_time=8.681e-05, forward_time=0.085, loss_ctc=13.575, loss=13.575, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.476e-04, train_time=0.214
[islpc50:0/3] 2022-06-06 16:44:54,852 (trainer:334) INFO: 2epoch results: [train] iter_time=1.008e-04, forward_time=0.086, loss_ctc=20.389, loss=20.389, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.303e-04, train_time=0.216, time=1 hour, 27 minutes and 3.05 seconds, total_count=48194, gpu_max_cached_mem_GB=5.582, [valid] loss_ctc=46.415, cer_ctc=0.248, loss_att=nan, acc=nan, cer=nan, wer=nan, loss=46.415, time=1 minute and 10 seconds, total_count=928, gpu_max_cached_mem_GB=5.582, [att_plot] time=8.86 seconds, total_count=0, gpu_max_cached_mem_GB=5.582
[islpc50:0/3] 2022-06-06 16:45:25,703 (trainer:382) INFO: The best model has been updated: train.loss, valid.loss
[islpc50:0/3] 2022-06-06 16:45:25,704 (trainer:268) INFO: 3/18epoch started. Estimated time to finish: 23 hours, 43 minutes and 10.3 seconds
[islpc50:0/3] 2022-06-06 16:49:51,782 (trainer:678) INFO: 3epoch:train:1-1204batch: iter_time=2.227e-04, forward_time=0.086, loss_ctc=12.420, loss=12.420, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.445e-04, train_time=0.221
[islpc50:0/3] 2022-06-06 16:54:14,445 (trainer:678) INFO: 3epoch:train:1205-2408batch: iter_time=8.502e-05, forward_time=0.087, loss_ctc=12.103, loss=12.103, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.415e-04, train_time=0.218
[islpc50:0/3] 2022-06-06 16:58:35,689 (trainer:678) INFO: 3epoch:train:2409-3612batch: iter_time=8.310e-05, forward_time=0.086, loss_ctc=12.193, loss=12.193, backward_time=0.054, optim_step_time=0.004, optim0_lr0=2.386e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 17:02:56,849 (trainer:678) INFO: 3epoch:train:3613-4816batch: iter_time=8.286e-05, forward_time=0.086, loss_ctc=11.578, loss=11.578, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.359e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 17:07:19,648 (trainer:678) INFO: 3epoch:train:4817-6020batch: iter_time=8.726e-05, forward_time=0.087, loss_ctc=11.342, loss=11.342, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.332e-04, train_time=0.218
[islpc50:0/3] 2022-06-06 17:11:43,714 (trainer:678) INFO: 3epoch:train:6021-7224batch: iter_time=8.949e-05, forward_time=0.088, loss_ctc=11.689, loss=11.689, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.306e-04, train_time=0.219
[islpc50:0/3] 2022-06-06 17:16:04,509 (trainer:678) INFO: 3epoch:train:7225-8428batch: iter_time=8.385e-05, forward_time=0.086, loss_ctc=10.432, loss=10.432, backward_time=0.052, optim_step_time=0.003, optim0_lr0=2.282e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 17:20:24,693 (trainer:678) INFO: 3epoch:train:8429-9632batch: iter_time=8.608e-05, forward_time=0.086, loss_ctc=10.984, loss=10.984, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.257e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 17:24:46,141 (trainer:678) INFO: 3epoch:train:9633-10836batch: iter_time=8.620e-05, forward_time=0.087, loss_ctc=11.074, loss=11.074, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.234e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 17:29:08,930 (trainer:678) INFO: 3epoch:train:10837-12040batch: iter_time=8.378e-05, forward_time=0.087, loss_ctc=10.429, loss=10.429, backward_time=0.054, optim_step_time=0.004, optim0_lr0=2.211e-04, train_time=0.218
[islpc50:0/3] 2022-06-06 17:33:29,327 (trainer:678) INFO: 3epoch:train:12041-13244batch: iter_time=8.856e-05, forward_time=0.086, loss_ctc=10.019, loss=10.019, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.189e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 17:37:48,915 (trainer:678) INFO: 3epoch:train:13245-14448batch: iter_time=8.410e-05, forward_time=0.086, loss_ctc=10.285, loss=10.285, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.168e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 17:42:07,265 (trainer:678) INFO: 3epoch:train:14449-15652batch: iter_time=8.806e-05, forward_time=0.085, loss_ctc=9.902, loss=9.902, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.147e-04, train_time=0.214
[islpc50:0/3] 2022-06-06 17:46:28,505 (trainer:678) INFO: 3epoch:train:15653-16856batch: iter_time=8.415e-05, forward_time=0.086, loss_ctc=9.820, loss=9.820, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.127e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 17:50:50,410 (trainer:678) INFO: 3epoch:train:16857-18060batch: iter_time=8.602e-05, forward_time=0.086, loss_ctc=9.419, loss=9.419, backward_time=0.055, optim_step_time=0.004, optim0_lr0=2.108e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 17:55:09,879 (trainer:678) INFO: 3epoch:train:18061-19264batch: iter_time=8.205e-05, forward_time=0.085, loss_ctc=8.891, loss=8.891, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.088e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 17:59:29,161 (trainer:678) INFO: 3epoch:train:19265-20468batch: iter_time=8.896e-05, forward_time=0.085, loss_ctc=8.897, loss=8.897, backward_time=0.052, optim_step_time=0.004, optim0_lr0=2.070e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 18:03:48,170 (trainer:678) INFO: 3epoch:train:20469-21672batch: iter_time=8.384e-05, forward_time=0.085, loss_ctc=9.254, loss=9.254, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.052e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 18:08:08,475 (trainer:678) INFO: 3epoch:train:21673-22876batch: iter_time=8.550e-05, forward_time=0.086, loss_ctc=8.949, loss=8.949, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.034e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 18:12:28,670 (trainer:678) INFO: 3epoch:train:22877-24080batch: iter_time=8.507e-05, forward_time=0.086, loss_ctc=8.746, loss=8.746, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.017e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 18:13:50,932 (trainer:334) INFO: 3epoch results: [train] iter_time=9.234e-05, forward_time=0.086, loss_ctc=10.421, loss=10.421, backward_time=0.053, optim_step_time=0.004, optim0_lr0=2.211e-04, train_time=0.217, time=1 hour, 27 minutes and 8.1 seconds, total_count=72291, gpu_max_cached_mem_GB=5.625, [valid] loss_ctc=39.748, cer_ctc=0.212, loss_att=nan, acc=nan, cer=nan, wer=nan, loss=39.748, time=1 minute and 8.57 seconds, total_count=1392, gpu_max_cached_mem_GB=5.625, [att_plot] time=8.56 seconds, total_count=0, gpu_max_cached_mem_GB=5.625
[islpc50:0/3] 2022-06-06 18:14:24,932 (trainer:382) INFO: The best model has been updated: train.loss, valid.loss
[islpc50:0/3] 2022-06-06 18:14:24,934 (trainer:268) INFO: 4/18epoch started. Estimated time to finish: 22 hours, 14 minutes and 25.09 seconds
[islpc50:0/3] 2022-06-06 18:18:53,210 (trainer:678) INFO: 4epoch:train:1-1204batch: iter_time=2.282e-04, forward_time=0.086, loss_ctc=7.242, loss=7.242, backward_time=0.054, optim_step_time=0.004, optim0_lr0=2.000e-04, train_time=0.223
[islpc50:0/3] 2022-06-06 18:23:13,670 (trainer:678) INFO: 4epoch:train:1205-2408batch: iter_time=9.354e-05, forward_time=0.086, loss_ctc=7.106, loss=7.106, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.984e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 18:27:35,604 (trainer:678) INFO: 4epoch:train:2409-3612batch: iter_time=8.760e-05, forward_time=0.087, loss_ctc=7.061, loss=7.061, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.968e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 18:31:56,430 (trainer:678) INFO: 4epoch:train:3613-4816batch: iter_time=8.588e-05, forward_time=0.086, loss_ctc=7.321, loss=7.321, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.952e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 18:36:17,638 (trainer:678) INFO: 4epoch:train:4817-6020batch: iter_time=9.013e-05, forward_time=0.087, loss_ctc=7.530, loss=7.530, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.937e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 18:40:37,880 (trainer:678) INFO: 4epoch:train:6021-7224batch: iter_time=8.538e-05, forward_time=0.086, loss_ctc=6.717, loss=6.717, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.922e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 18:44:59,460 (trainer:678) INFO: 4epoch:train:7225-8428batch: iter_time=8.532e-05, forward_time=0.086, loss_ctc=7.055, loss=7.055, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.908e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 18:49:19,779 (trainer:678) INFO: 4epoch:train:8429-9632batch: iter_time=8.768e-05, forward_time=0.086, loss_ctc=6.870, loss=6.870, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.894e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 18:53:40,505 (trainer:678) INFO: 4epoch:train:9633-10836batch: iter_time=8.557e-05, forward_time=0.086, loss_ctc=7.006, loss=7.006, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.880e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 18:58:00,226 (trainer:678) INFO: 4epoch:train:10837-12040batch: iter_time=8.638e-05, forward_time=0.085, loss_ctc=6.698, loss=6.698, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.866e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 19:02:22,913 (trainer:678) INFO: 4epoch:train:12041-13244batch: iter_time=8.433e-05, forward_time=0.087, loss_ctc=6.727, loss=6.727, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.853e-04, train_time=0.218
[islpc50:0/3] 2022-06-06 19:06:43,949 (trainer:678) INFO: 4epoch:train:13245-14448batch: iter_time=8.851e-05, forward_time=0.086, loss_ctc=7.100, loss=7.100, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.840e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 19:11:05,424 (trainer:678) INFO: 4epoch:train:14449-15652batch: iter_time=8.878e-05, forward_time=0.086, loss_ctc=6.964, loss=6.964, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.827e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 19:15:25,437 (trainer:678) INFO: 4epoch:train:15653-16856batch: iter_time=8.702e-05, forward_time=0.086, loss_ctc=6.492, loss=6.492, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.815e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 19:19:48,768 (trainer:678) INFO: 4epoch:train:16857-18060batch: iter_time=8.835e-05, forward_time=0.087, loss_ctc=6.376, loss=6.376, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.803e-04, train_time=0.219
[islpc50:0/3] 2022-06-06 19:24:08,370 (trainer:678) INFO: 4epoch:train:18061-19264batch: iter_time=8.527e-05, forward_time=0.085, loss_ctc=6.807, loss=6.807, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.791e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 19:28:29,614 (trainer:678) INFO: 4epoch:train:19265-20468batch: iter_time=8.991e-05, forward_time=0.086, loss_ctc=6.495, loss=6.495, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.779e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 19:32:50,419 (trainer:678) INFO: 4epoch:train:20469-21672batch: iter_time=8.685e-05, forward_time=0.086, loss_ctc=6.727, loss=6.727, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.767e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 19:37:09,585 (trainer:678) INFO: 4epoch:train:21673-22876batch: iter_time=8.472e-05, forward_time=0.085, loss_ctc=6.470, loss=6.470, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.756e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 19:41:30,914 (trainer:678) INFO: 4epoch:train:22877-24080batch: iter_time=8.583e-05, forward_time=0.086, loss_ctc=6.395, loss=6.395, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.745e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 19:42:51,681 (trainer:334) INFO: 4epoch results: [train] iter_time=9.428e-05, forward_time=0.086, loss_ctc=6.858, loss=6.858, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.864e-04, train_time=0.217, time=1 hour, 27 minutes and 11.03 seconds, total_count=96388, gpu_max_cached_mem_GB=5.625, [valid] loss_ctc=38.759, cer_ctc=0.198, loss_att=nan, acc=nan, cer=nan, wer=nan, loss=38.759, time=1 minute and 8.46 seconds, total_count=1856, gpu_max_cached_mem_GB=5.625, [att_plot] time=7.26 seconds, total_count=0, gpu_max_cached_mem_GB=5.625
[islpc50:0/3] 2022-06-06 19:43:24,514 (trainer:382) INFO: The best model has been updated: train.loss, valid.loss
[islpc50:0/3] 2022-06-06 19:43:24,516 (trainer:268) INFO: 5/18epoch started. Estimated time to finish: 20 hours, 45 minutes and 34.1 seconds
[islpc50:0/3] 2022-06-06 19:47:52,417 (trainer:678) INFO: 5epoch:train:1-1204batch: iter_time=2.279e-04, forward_time=0.086, loss_ctc=5.298, loss=5.298, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.734e-04, train_time=0.222
[islpc50:0/3] 2022-06-06 19:52:12,285 (trainer:678) INFO: 5epoch:train:1205-2408batch: iter_time=8.396e-05, forward_time=0.086, loss_ctc=4.989, loss=4.989, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.723e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 19:56:33,934 (trainer:678) INFO: 5epoch:train:2409-3612batch: iter_time=8.697e-05, forward_time=0.087, loss_ctc=5.368, loss=5.368, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.713e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 20:00:55,499 (trainer:678) INFO: 5epoch:train:3613-4816batch: iter_time=9.094e-05, forward_time=0.086, loss_ctc=4.841, loss=4.841, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.703e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 20:05:15,766 (trainer:678) INFO: 5epoch:train:4817-6020batch: iter_time=8.874e-05, forward_time=0.086, loss_ctc=5.039, loss=5.039, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.692e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 20:09:37,278 (trainer:678) INFO: 5epoch:train:6021-7224batch: iter_time=8.396e-05, forward_time=0.086, loss_ctc=5.092, loss=5.092, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.682e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 20:13:58,581 (trainer:678) INFO: 5epoch:train:7225-8428batch: iter_time=8.443e-05, forward_time=0.086, loss_ctc=5.165, loss=5.165, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.673e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 20:18:19,964 (trainer:678) INFO: 5epoch:train:8429-9632batch: iter_time=8.579e-05, forward_time=0.086, loss_ctc=5.251, loss=5.251, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.663e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 20:22:43,094 (trainer:678) INFO: 5epoch:train:9633-10836batch: iter_time=8.833e-05, forward_time=0.087, loss_ctc=5.247, loss=5.247, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.654e-04, train_time=0.218
[islpc50:0/3] 2022-06-06 20:27:02,629 (trainer:678) INFO: 5epoch:train:10837-12040batch: iter_time=8.460e-05, forward_time=0.086, loss_ctc=5.650, loss=5.650, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.644e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 20:31:24,443 (trainer:678) INFO: 5epoch:train:12041-13244batch: iter_time=8.576e-05, forward_time=0.086, loss_ctc=5.179, loss=5.179, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.635e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 20:35:46,762 (trainer:678) INFO: 5epoch:train:13245-14448batch: iter_time=8.361e-05, forward_time=0.086, loss_ctc=5.208, loss=5.208, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.626e-04, train_time=0.218
[islpc50:0/3] 2022-06-06 20:40:09,550 (trainer:678) INFO: 5epoch:train:14449-15652batch: iter_time=9.316e-05, forward_time=0.087, loss_ctc=4.893, loss=4.893, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.618e-04, train_time=0.218
[islpc50:0/3] 2022-06-06 20:44:29,045 (trainer:678) INFO: 5epoch:train:15653-16856batch: iter_time=8.705e-05, forward_time=0.086, loss_ctc=4.925, loss=4.925, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.609e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 20:48:48,671 (trainer:678) INFO: 5epoch:train:16857-18060batch: iter_time=9.016e-05, forward_time=0.086, loss_ctc=4.927, loss=4.927, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.600e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 20:53:09,438 (trainer:678) INFO: 5epoch:train:18061-19264batch: iter_time=8.686e-05, forward_time=0.086, loss_ctc=4.956, loss=4.956, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.592e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 20:57:29,460 (trainer:678) INFO: 5epoch:train:19265-20468batch: iter_time=8.588e-05, forward_time=0.086, loss_ctc=4.796, loss=4.796, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.584e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 21:01:50,863 (trainer:678) INFO: 5epoch:train:20469-21672batch: iter_time=9.126e-05, forward_time=0.086, loss_ctc=4.697, loss=4.697, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.576e-04, train_time=0.217
[islpc50:0/3] 2022-06-06 21:06:11,586 (trainer:678) INFO: 5epoch:train:21673-22876batch: iter_time=8.788e-05, forward_time=0.086, loss_ctc=4.532, loss=4.532, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.568e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 21:10:30,804 (trainer:678) INFO: 5epoch:train:22877-24080batch: iter_time=8.505e-05, forward_time=0.085, loss_ctc=4.885, loss=4.885, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.560e-04, train_time=0.215
[islpc50:0/3] 2022-06-06 21:11:52,063 (trainer:334) INFO: 5epoch results: [train] iter_time=9.412e-05, forward_time=0.086, loss_ctc=5.047, loss=5.047, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.642e-04, train_time=0.217, time=1 hour, 27 minutes and 11.53 seconds, total_count=120485, gpu_max_cached_mem_GB=5.625, [valid] loss_ctc=38.314, cer_ctc=0.187, loss_att=nan, acc=nan, cer=nan, wer=nan, loss=38.314, time=1 minute and 8.63 seconds, total_count=2320, gpu_max_cached_mem_GB=5.625, [att_plot] time=7.38 seconds, total_count=0, gpu_max_cached_mem_GB=5.625
[islpc50:0/3] 2022-06-06 21:12:21,803 (trainer:382) INFO: The best model has been updated: train.loss, valid.loss
[islpc50:0/3] 2022-06-06 21:12:21,804 (trainer:268) INFO: 6/18epoch started. Estimated time to finish: 19 hours, 16 minutes and 33.71 seconds
[islpc50:0/3] 2022-06-06 21:16:47,749 (trainer:678) INFO: 6epoch:train:1-1204batch: iter_time=2.335e-04, forward_time=0.086, loss_ctc=4.052, loss=4.052, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.552e-04, train_time=0.221
[islpc50:0/3] 2022-06-06 21:21:07,642 (trainer:678) INFO: 6epoch:train:1205-2408batch: iter_time=8.737e-05, forward_time=0.086, loss_ctc=3.812, loss=3.812, backward_time=0.053, optim_step_time=0.004, optim0_lr0=1.544e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 21:25:28,437 (trainer:678) INFO: 6epoch:train:2409-3612batch: iter_time=9.069e-05, forward_time=0.086, loss_ctc=4.229, loss=4.229, backward_time=0.054, optim_step_time=0.004, optim0_lr0=1.537e-04, train_time=0.216
[islpc50:0/3] 2022-06-06 21:29:52,010 (trainer:678) INFO: 6epoch:train:3613-4816batch: iter_time=8.376e-05, forward_time=0.088, loss_ctc=3.973, loss=3.973, backward_time=0.052, optim_step_time=0.004, optim0_lr0=1.529e-04, train_time=0.219
Traceback (most recent call last):
  File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "/project/ocean/junweih/espnet/espnet2/bin/asr_train.py", line 23, in <module>
    main()
  File "/project/ocean/junweih/espnet/espnet2/bin/asr_train.py", line 19, in main
    ASRTask.main(cmd=cmd)
  File "/project/ocean/junweih/espnet/espnet2/tasks/abs_task.py", line 1069, in main
    while not ProcessContext(processes, error_queues).join():
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 99, in join
    ready = multiprocessing.connection.wait(
  File "/usr/lib/python3.8/multiprocessing/connection.py", line 931, in wait
    ready = selector.select(timeout)
  File "/usr/lib/python3.8/selectors.py", line 415, in select
    fd_event_list = self._selector.poll(timeout)
KeyboardInterrupt
Process SpawnProcess-2:
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/project/ocean/junweih/espnet/espnet2/tasks/abs_task.py", line 1315, in main_worker
    cls.trainer.run(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 286, in run
    all_steps_are_invalid = cls.train_one_epoch(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 589, in train_one_epoch
    loss.backward()
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/_tensor.py", line 307, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/autograd/__init__.py", line 154, in backward
    Variable._execution_engine.run_backward(
KeyboardInterrupt
Process SpawnProcess-3:
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/project/ocean/junweih/espnet/espnet2/tasks/abs_task.py", line 1315, in main_worker
    cls.trainer.run(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 286, in run
    all_steps_are_invalid = cls.train_one_epoch(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 589, in train_one_epoch
    loss.backward()
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/_tensor.py", line 307, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/autograd/__init__.py", line 154, in backward
    Variable._execution_engine.run_backward(
KeyboardInterrupt
Process SpawnProcess-1:
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/project/ocean/junweih/espnet/espnet2/tasks/abs_task.py", line 1315, in main_worker
    cls.trainer.run(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 286, in run
    all_steps_are_invalid = cls.train_one_epoch(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 567, in train_one_epoch
    stats, weight = recursive_average(stats, weight, distributed)
  File "/project/ocean/junweih/espnet/espnet2/torch_utils/recursive_op.py", line 44, in recursive_average
    torch.distributed.all_reduce(weight, op=ReduceOp.SUM)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1292, in all_reduce
    work.wait()
KeyboardInterrupt
/usr/lib/python3.8/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 38 leaked semaphore objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '