aframson
/

RDPD-mini

Text Generation

feature-extraction

Model card Files Files and versions Community

aframson commited on Oct 21, 2023

Commit

0628687

·

1 Parent(s): 0aec99e

hj

Files changed (1) hide show

modelLM.py +1 -1

modelLM.py CHANGED Viewed

@@ -27,7 +27,7 @@ class OBILanguageModel(PreTrainedModel):
         pos_emb = self.position_embedding_table(torch.arange(input_ids.size(1), device='cpu'))
         x = tok_emb + pos_emb
-        x = self.transformer(x, x)
         x = self.ln1(x)
         x = self.ln2(x)
         logits = self.lm_head(x)

         pos_emb = self.position_embedding_table(torch.arange(input_ids.size(1), device='cpu'))
         x = tok_emb + pos_emb
+        x = self.transformer(x)
         x = self.ln1(x)
         x = self.ln2(x)
         logits = self.lm_head(x)