ltg
/

gpt-bert-babylm-small

Model card Files Files and versions Community

lgcharpe commited on Jan 27

Commit

691a253

·

verified ·

1 Parent(s): 606e14c

Update modeling_ltgbert.py

Files changed (1) hide show

modeling_ltgbert.py +1 -1

modeling_ltgbert.py CHANGED Viewed

@@ -230,7 +230,7 @@ class Attention(nn.Module):
         query = query.reshape(query_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
         key = key.reshape(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
-        value = value.view(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
         attention_scores = torch.bmm(query, key.transpose(1, 2) * self.scale)

         query = query.reshape(query_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
         key = key.reshape(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
+        value = value.reshape(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
         attention_scores = torch.bmm(query, key.transpose(1, 2) * self.scale)