Skip to content

Commit 09d9056

Browse files
committed
Apply code-format changes
Signed-off-by: Molly Sophia <[email protected]>
1 parent 3b4ec5e commit 09d9056

File tree

1 file changed

+7
-5
lines changed

1 file changed

+7
-5
lines changed

convert_hf_to_gguf.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3484,6 +3484,9 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
34843484
class Rwkv7Model(Rwkv6Model):
34853485
model_arch = gguf.MODEL_ARCH.RWKV7
34863486

3487+
def calc_lora_rank(self, hidden_size, exponent, multiplier):
3488+
return max(1, round(hidden_size ** exponent * multiplier / 32)) * 32
3489+
34873490
def set_gguf_parameters(self):
34883491
block_count = self.hparams["num_hidden_layers"]
34893492
head_size = self.hparams["head_size"]
@@ -3492,11 +3495,10 @@ def set_gguf_parameters(self):
34923495
intermediate_size = self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else (hidden_size * 4)
34933496

34943497
# ICLR: In-Context-Learning-Rate
3495-
calc_lora_rank = lambda exponent, multiplier: max(1, round(hidden_size ** exponent * multiplier / 32)) * 32
3496-
lora_rank_decay = self.hparams["lora_rank_decay"] if self.hparams["lora_rank_decay"] is not None else calc_lora_rank(0.5, 1.8)
3497-
lora_rank_iclr = self.hparams["lora_rank_iclr"] if self.hparams["lora_rank_iclr"] is not None else calc_lora_rank(0.5, 1.8)
3498-
lora_rank_value_residual_mix = self.hparams["lora_rank_value_residual_mix"] if self.hparams["lora_rank_value_residual_mix"] is not None else calc_lora_rank(0.5, 1.3)
3499-
lora_rank_gate = self.hparams["lora_rank_gate"] if self.hparams["lora_rank_gate"] is not None else calc_lora_rank(0.8, 0.6)
3498+
lora_rank_decay = self.hparams["lora_rank_decay"] if self.hparams["lora_rank_decay"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)
3499+
lora_rank_iclr = self.hparams["lora_rank_iclr"] if self.hparams["lora_rank_iclr"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)
3500+
lora_rank_value_residual_mix = self.hparams["lora_rank_value_residual_mix"] if self.hparams["lora_rank_value_residual_mix"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3)
3501+
lora_rank_gate = self.hparams["lora_rank_gate"] if self.hparams["lora_rank_gate"] is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6)
35003502

35013503
# RWKV isn't context limited
35023504
self.gguf_writer.add_context_length(1048576)

0 commit comments

Comments
 (0)