# python3 -m espnet2.bin.asr_train --use_preprocessor true --bpemodel data/cy_token_list/bpe_unigram150/bpe.model --token_type bpe --token_list data/cy_token_list/bpe_unigram150/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_cy/wav.scp,speech,sound --valid_data_path_and_name_and_type dump/raw/dev_cy/text,text,text --valid_shape_file exp/asr_stats_raw_cy_bpe150_sp/valid/speech_shape --valid_shape_file exp/asr_stats_raw_cy_bpe150_sp/valid/text_shape.bpe --resume true --init_param --ignore_init_mismatch false --fold_length 80000 --fold_length 150 --output_dir exp/asr_oxford_frontend_raw_cy_bpe150_sp --config conf/tuning/oxford_frontend.yaml --frontend_conf fs=16k --train_data_path_and_name_and_type dump/raw/train_cy_sp/wav.scp,speech,sound --train_data_path_and_name_and_type dump/raw/train_cy_sp/text,text,text --train_shape_file exp/asr_stats_raw_cy_bpe150_sp/train/speech_shape --train_shape_file exp/asr_stats_raw_cy_bpe150_sp/train/text_shape.bpe --ngpu 3 --multiprocessing_distributed True 
# Started at Mon Jun  6 13:43:54 EDT 2022
#
/usr/bin/python3 /project/ocean/junweih/espnet/espnet2/bin/asr_train.py --use_preprocessor true --bpemodel data/cy_token_list/bpe_unigram150/bpe.model --token_type bpe --token_list data/cy_token_list/bpe_unigram150/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_cy/wav.scp,speech,sound --valid_data_path_and_name_and_type dump/raw/dev_cy/text,text,text --valid_shape_file exp/asr_stats_raw_cy_bpe150_sp/valid/speech_shape --valid_shape_file exp/asr_stats_raw_cy_bpe150_sp/valid/text_shape.bpe --resume true --init_param --ignore_init_mismatch false --fold_length 80000 --fold_length 150 --output_dir exp/asr_oxford_frontend_raw_cy_bpe150_sp --config conf/tuning/oxford_frontend.yaml --frontend_conf fs=16k --train_data_path_and_name_and_type dump/raw/train_cy_sp/wav.scp,speech,sound --train_data_path_and_name_and_type dump/raw/train_cy_sp/text,text,text --train_shape_file exp/asr_stats_raw_cy_bpe150_sp/train/speech_shape --train_shape_file exp/asr_stats_raw_cy_bpe150_sp/train/text_shape.bpe --ngpu 3 --multiprocessing_distributed True
[islpc50:0/3] 2022-06-06 13:44:05,310 (distributed_c10d:217) INFO: Added key: store_based_barrier_key:1 to store for rank: 0
[islpc50:0/3] 2022-06-06 13:44:05,310 (distributed_c10d:251) INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 3 nodes.
[islpc50:0/3] 2022-06-06 13:44:05,357 (asr:411) INFO: Vocabulary size: 150
[islpc50:0/3] 2022-06-06 13:44:05,863 (filelock:274) INFO: Lock 139812231996992 acquired on ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f.lock
[islpc50:0/3] 2022-06-06 13:44:05,864 (filelock:318) INFO: Lock 139812231996992 released on ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f.lock
[Featurizer] - The selected feature last_hidden_state's downsample rate is 320
[islpc50:0/3] 2022-06-06 13:44:20,588 (s3prl:159) INFO: Pretrained S3PRL frontend model parameters reloaded!
[islpc50:0/3] 2022-06-06 13:44:24,553 (abs_task:1157) INFO: pytorch.version=1.10.1+cu111, cuda.available=True, cudnn.version=8005, cudnn.benchmark=False, cudnn.deterministic=True
[islpc50:0/3] 2022-06-06 13:44:24,558 (abs_task:1158) INFO: Model structure:
ESPnetASRModel(
  (frontend): S3prlFrontend(
    (upstream): UpstreamExpert(
      (model): Wav2Vec2Model(
        (feature_extractor): ConvFeatureExtractionModel(
          (conv_layers): ModuleList(
            (0): Sequential(
              (0): Conv1d(1, 512, kernel_size=(10,), stride=(5,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (1): Sequential(
              (0): Conv1d(512, 512, kernel_size=(3,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (2): Sequential(
              (0): Conv1d(512, 512, kernel_size=(3,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (3): Sequential(
              (0): Conv1d(512, 512, kernel_size=(3,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (4): Sequential(
              (0): Conv1d(512, 512, kernel_size=(3,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (5): Sequential(
              (0): Conv1d(512, 512, kernel_size=(2,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
            (6): Sequential(
              (0): Conv1d(512, 512, kernel_size=(2,), stride=(2,))
              (1): Dropout(p=0.0, inplace=False)
              (2): Sequential(
                (0): TransposeLast()
                (1): Fp32LayerNorm((512,), eps=1e-05, elementwise_affine=True)
                (2): TransposeLast()
              )
              (3): GELU()
            )
          )
        )
        (post_extract_proj): Linear(in_features=512, out_features=1024, bias=True)
        (dropout_input): Dropout(p=0.1, inplace=False)
        (dropout_features): Dropout(p=0.1, inplace=False)
        (quantizer): GumbelVectorQuantizer(
          (weight_proj): Linear(in_features=512, out_features=640, bias=True)
        )
        (project_q): Linear(in_features=768, out_features=768, bias=True)
        (encoder): TransformerEncoder(
          (pos_conv): Sequential(
            (0): Conv1d(1024, 1024, kernel_size=(128,), stride=(1,), padding=(64,), groups=16)
            (1): SamePad()
            (2): GELU()
          )
          (layers): ModuleList(
            (0): AdapterTransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (adapter1): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
              (adapter2): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
            )
            (1): AdapterTransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (adapter1): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
              (adapter2): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
            )
            (2): AdapterTransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (adapter1): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
              (adapter2): Adapter(
                (down_projection): Linear(in_features=1024, out_features=192, bias=True)
                (up_projection): Linear(in_features=192, out_features=1024, bias=True)
              )
            )
            (3): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (4): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (5): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (6): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (7): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (8): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (9): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (10): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (11): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (12): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (13): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (14): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (15): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (16): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (17): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (18): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (19): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (20): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (21): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (22): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
            (23): TransformerSentenceEncoderLayer(
              (self_attn): MultiheadAttention(
                (dropout_module): FairseqDropout()
                (k_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (v_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (q_proj): Linear(in_features=1024, out_features=1024, bias=True)
                (out_proj): Linear(in_features=1024, out_features=1024, bias=True)
              )
              (dropout1): Dropout(p=0.0, inplace=False)
              (dropout2): Dropout(p=0.0, inplace=False)
              (dropout3): Dropout(p=0.0, inplace=False)
              (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
              (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
            )
          )
          (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
        )
        (layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
        (final_proj): Linear(in_features=1024, out_features=768, bias=True)
      )
    )
    (featurizer): Featurizer()
  )
  (normalize): UtteranceMVN(norm_means=True, norm_vars=False)
  (encoder): TransformerEncoder(
    (embed): Conv2dSubsampling(
      (conv): Sequential(
        (0): Conv2d(1, 1024, kernel_size=(3, 3), stride=(2, 2))
        (1): ReLU()
        (2): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(2, 2))
        (3): ReLU()
      )
      (out): Sequential(
        (0): Linear(in_features=261120, out_features=1024, bias=True)
        (1): PositionalEncoding(
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
    )
    (encoders): MultiSequential(
      (0): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
          (dropout): Dropout(p=0.0, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=1024, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=1024, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (1): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
          (dropout): Dropout(p=0.0, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=1024, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=1024, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (2): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
          (dropout): Dropout(p=0.0, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=1024, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=1024, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (3): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
          (dropout): Dropout(p=0.0, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=1024, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=1024, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (4): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
          (dropout): Dropout(p=0.0, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=1024, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=1024, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (5): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
          (dropout): Dropout(p=0.0, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=1024, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=1024, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
    )
    (after_norm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
  )
  (criterion_att): LabelSmoothingLoss(
    (criterion): KLDivLoss()
  )
  (ctc): CTC(
    (ctc_lo): Linear(in_features=1024, out_features=150, bias=True)
    (ctc_loss): CTCLoss()
  )
)

Model summary:
    Class Name: ESPnetASRModel
    Total Number of model parameters: 647.15 M
    Number of trainable parameters: 329.76 M (51.0%)
    Size: 1.32 GB
    Type: torch.float32
[islpc50:0/3] 2022-06-06 13:44:24,559 (abs_task:1161) INFO: Optimizer:
Adam (
Parameter Group 0
    amsgrad: False
    betas: (0.9, 0.999)
    eps: 1e-08
    initial_lr: 0.00027
    lr: 6.749999999999999e-09
    weight_decay: 0
)
[islpc50:0/3] 2022-06-06 13:44:24,559 (abs_task:1162) INFO: Scheduler: WarmupLR(warmup_steps=40000)
[islpc50:0/3] 2022-06-06 13:44:24,564 (abs_task:1171) INFO: Saving the configuration in exp/asr_oxford_frontend_raw_cy_bpe150_sp/config.yaml
[islpc50:0/3] 2022-06-06 13:44:26,790 (abs_task:1525) INFO: [train] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/train_cy_sp/wav.scp", "type": "sound"}
  text: {"path": "dump/raw/train_cy_sp/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.CommonPreprocessor object at 0x7f28098a80a0>)
[islpc50:0/3] 2022-06-06 13:44:26,790 (abs_task:1526) INFO: [train] Batch sampler: FoldedBatchSampler(N-batch=58673, batch_size=1, shape_files=['exp/asr_stats_raw_cy_bpe150_sp/train/speech_shape', 'exp/asr_stats_raw_cy_bpe150_sp/train/text_shape.bpe'], sort_in_batch=descending, sort_batch=descending)
[islpc50:0/3] 2022-06-06 13:44:26,797 (abs_task:1527) INFO: [train] mini-batch sizes summary: N-batch=58673, mean=3.0, min=3, max=3
[islpc50:0/3] 2022-06-06 13:44:26,935 (abs_task:1525) INFO: [valid] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/dev_cy/wav.scp", "type": "sound"}
  text: {"path": "dump/raw/dev_cy/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.CommonPreprocessor object at 0x7f28098a83d0>)
[islpc50:0/3] 2022-06-06 13:44:26,935 (abs_task:1526) INFO: [valid] Batch sampler: FoldedBatchSampler(N-batch=977, batch_size=1, shape_files=['exp/asr_stats_raw_cy_bpe150_sp/valid/speech_shape', 'exp/asr_stats_raw_cy_bpe150_sp/valid/text_shape.bpe'], sort_in_batch=descending, sort_batch=descending)
[islpc50:0/3] 2022-06-06 13:44:26,935 (abs_task:1527) INFO: [valid] mini-batch sizes summary: N-batch=977, mean=3.0, min=3, max=4
[islpc50:0/3] 2022-06-06 13:44:26,972 (abs_task:1525) INFO: [plot_att] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/dev_cy/wav.scp", "type": "sound"}
  text: {"path": "dump/raw/dev_cy/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.CommonPreprocessor object at 0x7f27e17ac970>)
[islpc50:0/3] 2022-06-06 13:44:26,972 (abs_task:1526) INFO: [plot_att] Batch sampler: UnsortedBatchSampler(N-batch=2933, batch_size=1, key_file=exp/asr_stats_raw_cy_bpe150_sp/valid/speech_shape, 
[islpc50:0/3] 2022-06-06 13:44:26,972 (abs_task:1527) INFO: [plot_att] mini-batch sizes summary: N-batch=3, mean=1.0, min=1, max=1
islpc50:1025746:1025746 [0] NCCL INFO Bootstrap : Using bond0:128.2.205.9<0>
islpc50:1025746:1025746 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
islpc50:1025746:1025746 [0] NCCL INFO NET/IB : No device found.
islpc50:1025746:1025746 [0] NCCL INFO NET/Socket : Using [0]bond0:128.2.205.9<0>
islpc50:1025746:1025746 [0] NCCL INFO Using network Socket
NCCL version 2.10.3+cuda11.1
islpc50:1025747:1025747 [1] NCCL INFO Bootstrap : Using bond0:128.2.205.9<0>
islpc50:1025748:1025748 [2] NCCL INFO Bootstrap : Using bond0:128.2.205.9<0>
islpc50:1025747:1025747 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
islpc50:1025748:1025748 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
islpc50:1025747:1025747 [1] NCCL INFO NET/IB : No device found.
islpc50:1025748:1025748 [2] NCCL INFO NET/IB : No device found.
islpc50:1025747:1025747 [1] NCCL INFO NET/Socket : Using [0]bond0:128.2.205.9<0>
islpc50:1025748:1025748 [2] NCCL INFO NET/Socket : Using [0]bond0:128.2.205.9<0>
islpc50:1025747:1025747 [1] NCCL INFO Using network Socket
islpc50:1025748:1025748 [2] NCCL INFO Using network Socket
islpc50:1025746:1025832 [0] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025746:1025832 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025746:1025832 [0] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025746:1025832 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025748:1025834 [2] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025748:1025834 [2] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025748:1025834 [2] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025748:1025834 [2] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025747:1025833 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025747:1025833 [1] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025747:1025833 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025747:1025833 [1] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025747:1025833 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0
islpc50:1025747:1025833 [1] NCCL INFO Setting affinity for GPU 1 to ffffff
islpc50:1025748:1025834 [2] NCCL INFO Trees [0] -1/-1/-1->2->1 [1] -1/-1/-1->2->1
islpc50:1025746:1025832 [0] NCCL INFO Channel 00/02 :    0   1   2
islpc50:1025748:1025834 [2] NCCL INFO Setting affinity for GPU 2 to ffffff
islpc50:1025746:1025832 [0] NCCL INFO Channel 01/02 :    0   1   2
islpc50:1025746:1025832 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1
islpc50:1025746:1025832 [0] NCCL INFO Setting affinity for GPU 0 to ffffff
islpc50:1025747:1025833 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025747:1025833 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025747:1025833 [1] NCCL INFO Channel 00 : 1[1a000] -> 2[67000] via direct shared memory
islpc50:1025747:1025833 [1] NCCL INFO Channel 01 : 1[1a000] -> 2[67000] via direct shared memory
islpc50:1025746:1025832 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025748:1025834 [2] NCCL INFO Channel 00 : 2[67000] -> 0[19000] via direct shared memory
islpc50:1025746:1025832 [0] NCCL INFO Channel 00 : 0[19000] -> 1[1a000] via direct shared memory
islpc50:1025746:1025832 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025748:1025834 [2] NCCL INFO Channel 01 : 2[67000] -> 0[19000] via direct shared memory
islpc50:1025746:1025832 [0] NCCL INFO Channel 01 : 0[19000] -> 1[1a000] via direct shared memory
islpc50:1025748:1025834 [2] NCCL INFO Connected all rings
islpc50:1025747:1025833 [1] NCCL INFO Connected all rings
islpc50:1025746:1025832 [0] NCCL INFO Connected all rings
islpc50:1025748:1025834 [2] NCCL INFO Channel 00 : 2[67000] -> 1[1a000] via direct shared memory
islpc50:1025746:1025832 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025748:1025834 [2] NCCL INFO Channel 01 : 2[67000] -> 1[1a000] via direct shared memory
islpc50:1025746:1025832 [0] NCCL INFO Could not enable P2P between dev 0(=19000) and dev 1(=1a000)
islpc50:1025747:1025833 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025747:1025833 [1] NCCL INFO Channel 00 : 1[1a000] -> 0[19000] via direct shared memory
islpc50:1025747:1025833 [1] NCCL INFO Could not enable P2P between dev 1(=1a000) and dev 0(=19000)
islpc50:1025747:1025833 [1] NCCL INFO Channel 01 : 1[1a000] -> 0[19000] via direct shared memory
islpc50:1025746:1025832 [0] NCCL INFO Connected all trees
islpc50:1025746:1025832 [0] NCCL INFO threadThresholds 8/8/64 | 24/8/64 | 8/8/512
islpc50:1025746:1025832 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
islpc50:1025747:1025833 [1] NCCL INFO Connected all trees
islpc50:1025747:1025833 [1] NCCL INFO threadThresholds 8/8/64 | 24/8/64 | 8/8/512
islpc50:1025747:1025833 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
islpc50:1025748:1025834 [2] NCCL INFO Connected all trees
islpc50:1025748:1025834 [2] NCCL INFO threadThresholds 8/8/64 | 24/8/64 | 8/8/512
islpc50:1025748:1025834 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
islpc50:1025746:1025832 [0] NCCL INFO comm 0x7f27d0002fb0 rank 0 nranks 3 cudaDev 0 busId 19000 - Init COMPLETE
islpc50:1025747:1025833 [1] NCCL INFO comm 0x7f6e2c002fb0 rank 1 nranks 3 cudaDev 1 busId 1a000 - Init COMPLETE
islpc50:1025748:1025834 [2] NCCL INFO comm 0x7f6680002fb0 rank 2 nranks 3 cudaDev 2 busId 67000 - Init COMPLETE
islpc50:1025746:1025746 [0] NCCL INFO Launch mode Parallel
[islpc50:0/3] 2022-06-06 13:44:27,683 (trainer:280) INFO: 1/18epoch started
[s3prl.upstream.experts] Warning: can not import s3prl.upstream.byol_a.expert: No module named 'easydict'. Pass.
[s3prl.hub] Warning: can not import s3prl.upstream.byol_a.hubconf: No module named 'easydict'. Please see upstream/byol_a/README.md
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.quesst14_dtw.expert: No module named 'dtw'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.separation_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.enhancement_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.speech_commands.expert: No module named 'catalyst'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.a2a-vc-vctk.expert: No module named 'resemblyzer'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.voxceleb2_ge2e.expert: No module named 'sox'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.sv_voxceleb1.expert: No module named 'sox'. Pass.
Using cache found in ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f
for https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_new.pt
>> inserted adapters to the following layers: 0, 1, 2
  * original model weights: 317,390,592
  * new model weights - all: 319,757,184
  * new model weights - trainable: 2,366,592 ( 0.75% of original model)
[s3prl.upstream.experts] Warning: can not import s3prl.upstream.byol_a.expert: No module named 'easydict'. Pass.
[s3prl.hub] Warning: can not import s3prl.upstream.byol_a.hubconf: No module named 'easydict'. Please see upstream/byol_a/README.md
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.quesst14_dtw.expert: No module named 'dtw'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.separation_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.enhancement_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.speech_commands.expert: No module named 'catalyst'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.a2a-vc-vctk.expert: No module named 'resemblyzer'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.voxceleb2_ge2e.expert: No module named 'sox'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.sv_voxceleb1.expert: No module named 'sox'. Pass.
Using cache found in ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f
for https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_new.pt
>> inserted adapters to the following layers: 0, 1, 2
  * original model weights: 317,390,592
  * new model weights - all: 319,757,184
  * new model weights - trainable: 2,366,592 ( 0.75% of original model)
[s3prl.upstream.experts] Warning: can not import s3prl.upstream.byol_a.expert: No module named 'easydict'. Pass.
[s3prl.hub] Warning: can not import s3prl.upstream.byol_a.hubconf: No module named 'easydict'. Please see upstream/byol_a/README.md
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.quesst14_dtw.expert: No module named 'dtw'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.separation_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.enhancement_stft.expert: No module named 'asteroid'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.speech_commands.expert: No module named 'catalyst'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.a2a-vc-vctk.expert: No module named 'resemblyzer'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.voxceleb2_ge2e.expert: No module named 'sox'. Pass.
[s3prl.downstream.experts] Warning: can not import s3prl.downstream.sv_voxceleb1.expert: No module named 'sox'. Pass.
Using cache found in ./hub/s3prl_cache/1c76d6e88090f01736036b28dc995fef583f47f42662d55286332557f957609f
for https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_new.pt
>> inserted adapters to the following layers: 0, 1, 2
  * original model weights: 317,390,592
  * new model weights - all: 319,757,184
  * new model weights - trainable: 2,366,592 ( 0.75% of original model)
Process SpawnProcess-1:
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/project/ocean/junweih/espnet/espnet2/tasks/abs_task.py", line 1315, in main_worker
    cls.trainer.run(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 286, in run
    all_steps_are_invalid = cls.train_one_epoch(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 652, in train_one_epoch
    optimizer.step()
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/lr_scheduler.py", line 65, in wrapper
    return wrapped(*args, **kwargs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/optimizer.py", line 88, in wrapper
    return func(*args, **kwargs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/autograd/grad_mode.py", line 28, in decorate_context
    return func(*args, **kwargs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/adam.py", line 133, in step
    F.adam(params_with_grad,
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/_functional.py", line 94, in adam
    denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
RuntimeError: CUDA out of memory. Tried to allocate 1020.00 MiB (GPU 0; 10.76 GiB total capacity; 8.37 GiB already allocated; 554.56 MiB free; 8.68 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
Process SpawnProcess-2:
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/project/ocean/junweih/espnet/espnet2/tasks/abs_task.py", line 1315, in main_worker
    cls.trainer.run(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 286, in run
    all_steps_are_invalid = cls.train_one_epoch(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 652, in train_one_epoch
    optimizer.step()
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/lr_scheduler.py", line 65, in wrapper
    return wrapped(*args, **kwargs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/optimizer.py", line 88, in wrapper
    return func(*args, **kwargs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/autograd/grad_mode.py", line 28, in decorate_context
    return func(*args, **kwargs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/adam.py", line 133, in step
    F.adam(params_with_grad,
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/_functional.py", line 94, in adam
    denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
RuntimeError: CUDA out of memory. Tried to allocate 1020.00 MiB (GPU 1; 10.76 GiB total capacity; 8.37 GiB already allocated; 554.56 MiB free; 8.68 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
Process SpawnProcess-3:
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/project/ocean/junweih/espnet/espnet2/tasks/abs_task.py", line 1315, in main_worker
    cls.trainer.run(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 286, in run
    all_steps_are_invalid = cls.train_one_epoch(
  File "/project/ocean/junweih/espnet/espnet2/train/trainer.py", line 652, in train_one_epoch
    optimizer.step()
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/lr_scheduler.py", line 65, in wrapper
    return wrapped(*args, **kwargs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/optimizer.py", line 88, in wrapper
    return func(*args, **kwargs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/autograd/grad_mode.py", line 28, in decorate_context
    return func(*args, **kwargs)
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/adam.py", line 133, in step
    F.adam(params_with_grad,
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/optim/_functional.py", line 94, in adam
    denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
RuntimeError: CUDA out of memory. Tried to allocate 1020.00 MiB (GPU 2; 10.76 GiB total capacity; 8.37 GiB already allocated; 554.56 MiB free; 8.68 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
Traceback (most recent call last):
  File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "/project/ocean/junweih/espnet/espnet2/bin/asr_train.py", line 23, in <module>
    main()
  File "/project/ocean/junweih/espnet/espnet2/bin/asr_train.py", line 19, in main
    ASRTask.main(cmd=cmd)
  File "/project/ocean/junweih/espnet/espnet2/tasks/abs_task.py", line 1069, in main
    while not ProcessContext(processes, error_queues).join():
  File "/project/ocean/junweih/espnet/tools/python_user_base/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 139, in join
    raise ProcessExitedException(
torch.multiprocessing.spawn.ProcessExitedException: process 2 terminated with exit code 1
/usr/lib/python3.8/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 3 leaked semaphore objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
# Accounting: time=44 threads=1
# Ended (code 1) at Mon Jun  6 13:44:38 EDT 2022, elapsed time 44 seconds