convert-*.py: consolidate source_hf_repo and use hf params as fallback if present

Author: mofosyne

convert-hf-to-gguf.py

@@ -133,17 +133,22 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path,
                 logger.info(f"choosing --outtype bf16 from first tensor type ({first_tensor.dtype})")
                 self.ftype = gguf.LlamaFileType.MOSTLY_BF16
 
+        # Update any missing authorship metadata with huggingface_parameters
+        if self.metadata is not None and self.metadata.source_hf_repo is None:
+            if self.hparams is not None and "_name_or_path" in self.hparams:
+                self.metadata.source_hf_repo = self.hparams["_name_or_path"]
+
         # Set model name based on latest metadata either provided or calculated from environment
-        def get_model_name(metadata, hyperparameters, dir_model, model_arch):
+        def get_model_name(metadata, huggingface_parameters, dir_model, model_arch):
             if metadata is not None and metadata.name is not None:
                 # Explicit Metadata Was Provided By User
                 return metadata.name
-            elif hyperparameters is not None and "_name_or_path" in hyperparameters:
-                # Hugging Face Hyperparameter Model Name or Model Folder Name is Provided
-                return hyperparameters["_name_or_path"]
-            elif hyperparameters is not None and "model_type" in hyperparameters:
-                # Hugging Face Hyperparameter Model Type is Provided
-                return hyperparameters["model_type"]
+            elif huggingface_parameters is not None and "_name_or_path" in huggingface_parameters:
+                # Hugging Face Parameters Model Name or Model Folder Name is Provided
+                return huggingface_parameters["_name_or_path"]
+            elif huggingface_parameters is not None and "model_type" in huggingface_parameters:
+                # Hugging Face Parameters Model Type is Provided
+                return huggingface_parameters["model_type"]
             elif dir_model is not None and dir_model.name is not None:
                 # Use directory folder name
                 return dir_model.name
@@ -154,7 +159,7 @@ def get_model_name(metadata, hyperparameters, dir_model, model_arch):
         # Extracts and converts the encoding scheme from the given file type name. e.g. 'gguf.LlamaFileType.ALL_F32' --> 'F32'
         encodingScheme = self.ftype.name.partition("_")[2]
 
-        # Get Expert Count From Hyperparameters
+        # Get Expert Count From huggingface_parameters
         expert_count = self.hparams["num_local_experts"] if "num_local_experts" in self.hparams else None
 
         # Generate default filename based on model specification and available metadata
@@ -952,7 +957,6 @@ def set_gguf_parameters(self):
         block_count = self.hparams["num_hidden_layers"]
         head_count = self.hparams["num_attention_heads"]
         head_count_kv = self.hparams.get("num_key_value_heads", head_count)
-        hf_repo = self.hparams.get("_name_or_path", "")
 
         ctx_length = 0
         if "max_sequence_length" in self.hparams:
@@ -965,7 +969,6 @@ def set_gguf_parameters(self):
             raise ValueError("gguf: can not find ctx length parameter.")
 
         self.gguf_writer.add_file_type(self.ftype)
-        self.gguf_writer.add_source_hf_repo(hf_repo)
         self.gguf_writer.add_tensor_data_layout("Meta AI original pth")
         self.gguf_writer.add_context_length(ctx_length)
         self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
@@ -989,7 +992,6 @@ def set_gguf_parameters(self):
         block_count = self.hparams["num_hidden_layers"]
         head_count = self.hparams["num_attention_heads"]
         head_count_kv = self.hparams.get("num_key_value_heads", head_count)
-        hf_repo = self.hparams.get("_name_or_path", "")
 
         ctx_length = 0
         if "max_sequence_length" in self.hparams:
@@ -1001,7 +1003,6 @@ def set_gguf_parameters(self):
         else:
             raise ValueError("gguf: can not find ctx length parameter.")
 
-        self.gguf_writer.add_source_hf_repo(hf_repo)
         self.gguf_writer.add_tensor_data_layout("Meta AI original pth")
         self.gguf_writer.add_context_length(ctx_length)
         self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
@@ -1111,7 +1112,6 @@ def set_gguf_parameters(self):
         block_count = self.hparams["num_hidden_layers"]
         head_count = self.hparams["num_attention_heads"]
         head_count_kv = self.hparams.get("num_key_value_heads", head_count)
-        hf_repo = self.hparams.get("_name_or_path", "")
 
         ctx_length = 0
         if "max_sequence_length" in self.hparams:
@@ -1123,7 +1123,6 @@ def set_gguf_parameters(self):
         else:
             raise ValueError("gguf: can not find ctx length parameter.")
 
-        self.gguf_writer.add_source_hf_repo(hf_repo)
         self.gguf_writer.add_tensor_data_layout("Meta AI original pth")
         self.gguf_writer.add_context_length(ctx_length)
         self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])