Upload modeling_opt.py

modeling_opt.py

@@ -190,13 +190,13 @@ class OPTAttention(nn.Module):
         self.num_heads = config.num_attention_heads
         self.dropout = config.attention_dropout
         self.enable_bias = config.enable_bias
-        self.head_dim = embed_dim // num_heads
+        self.head_dim = self.embed_dim // self.num_heads
         self.is_causal = True
 
-
         if (self.head_dim * num_heads) != self.embed_dim:
             raise ValueError(
-                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {
+                    self.embed_dim}"
                 f" and `num_heads`: {num_heads})."
             )
         self.scaling = self.head_dim**-0.5
@@ -366,14 +366,16 @@ class OPTAttention(nn.Module):
 
         if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):
             raise ValueError(
-                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
+                f"Attention weights should be of size {
+                    (bsz * self.num_heads, tgt_len, src_len)}, but is"
                 f" {attn_weights.size()}"
             )
 
         if attention_mask is not None:
             if attention_mask.size() != (bsz, 1, tgt_len, src_len):
                 raise ValueError(
-                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}"
+                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {
+                        attention_mask.size()}"
                 )
             attn_weights = attn_weights.view(
                 bsz, self.num_heads, tgt_len, src_len) + attention_mask
@@ -394,7 +396,8 @@ class OPTAttention(nn.Module):
         if layer_head_mask is not None:
             if layer_head_mask.size() != (self.num_heads,):
                 raise ValueError(
-                    f"Head mask for a single layer should be of size {(self.num_heads,)}, but is"
+                    f"Head mask for a single layer should be of size {
+                        (self.num_heads,)}, but is"
                     f" {layer_head_mask.size()}"
                 )
             attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(
@@ -428,7 +431,8 @@ class OPTAttention(nn.Module):
 
         if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):
             raise ValueError(
-                f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is"
+                f"`attn_output` should be of size {
+                    (bsz, self.num_heads, tgt_len, self.head_dim)}, but is"
                 f" {attn_output.size()}"
             )
 
@@ -1086,7 +1090,8 @@ class OPTDecoder(OPTPreTrainedModel):
                     batch_size, mask_seq_length, device=inputs_embeds.device)
             elif attention_mask.shape[1] != mask_seq_length:
                 raise ValueError(
-                    f"The provided attention mask has length {attention_mask.shape[1]}, but its length should be "
+                    f"The provided attention mask has length {
+                        attention_mask.shape[1]}, but its length should be "
                     f"{mask_seq_length} (sum of the lengths of current and past inputs)"
                 )
             causal_attention_mask = _prepare_4d_causal_attention_mask(
@@ -1118,7 +1123,8 @@ class OPTDecoder(OPTPreTrainedModel):
             if attn_mask is not None:
                 if attn_mask.size()[0] != (len(self.layers)):
                     raise ValueError(
-                        f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"
+                        f"The `{mask_name}` should be specified for {
+                            len(self.layers)} layers, but it is for"
                         f" {head_mask.size()[0]}."
                     )