Adding back query pre attn scalar override.

michaelmoynihan · michaelmoynihan · commit a88abd52349d · 2024-07-08T23:56:54.000Z
diff --git a/gemma/config.py b/gemma/config.py
@@ -117,6 +117,7 @@ def get_config_for_9b() -> GemmaConfig:
         head_dim=256,
         attn_types=[AttentionType.LOCAL_SLIDING, AttentionType.GLOBAL] * 21,
         sliding_window_size=4096,
+        query_pre_attn_scalar=224, # hidden_size / num_attention_heads
     )
 
 

Original file line number	Diff line number	Diff line change
`@@ -117,6 +117,7 @@ def get_config_for_9b() -> GemmaConfig:`
`117`	`117`	`head_dim=256,`
`118`	`118`	`attn_types=[AttentionType.LOCAL_SLIDING, AttentionType.GLOBAL] * 21,`
`119`	`119`	`sliding_window_size=4096,`
	`120`	`+ query_pre_attn_scalar=224, # hidden_size / num_attention_heads`
`120`	`121`	`)`
`121`	`122`
`122`	`123`