Remove trailing whitespaces

Author: Igoorx

convert.py

@@ -183,7 +183,7 @@ def __init__(self, fname_tokenizer: Path, fname_added_tokens: Optional[Path], fn
             "eos_token": self.sentencepiece_tokenizer.eos_id(),
             "pad_token": self.sentencepiece_tokenizer.pad_id()
         }
-        
+
         tokenizer_config: Dict[str, Any]
         if fname_tokenizer_config is not None:
             tokenizer_config = json.load(open(fname_tokenizer_config))
@@ -194,7 +194,7 @@ def __init__(self, fname_tokenizer: Path, fname_added_tokens: Optional[Path], fn
             if key not in TOKEN_NAME_TO_ID or TOKEN_NAME_TO_ID[key] == -1:
                 continue
             self.special_tokens_map[TOKEN_NAME_TO_ID[key]] = value["content"] if isinstance(value, dict) else value
-        
+
         special_tokens: Dict[str, Any]
         if fname_special_tokens is not None:
             special_tokens = json.load(open(fname_special_tokens))
@@ -208,7 +208,7 @@ def __init__(self, fname_tokenizer: Path, fname_added_tokens: Optional[Path], fn
             if token_id == -1 or token_id in self.special_tokens_map:
                 continue
             self.special_tokens_map[token_id] = value["content"] if isinstance(value, dict) else value
-        
+
         self.vocab_special_size: int = len(self.added_tokens_list) + len(self.special_tokens_map)
 
     def sentencepiece_tokens(self) -> Iterable[Tuple[bytes, float]]:

llama-util.h

@@ -491,7 +491,7 @@ typedef llama_buffer llama_ctx_buffer;
 
 struct llama_trie_node {
     llama_trie_node(): is_terminator(false) {}
-    
+
     std::unordered_map<char, llama_trie_node*> children;
     bool is_terminator;
 };
@@ -506,7 +506,7 @@ struct llama_trie {
         if (word.empty()) {
             return;
         }
-            
+
         llama_trie_node *ref = root_;
         for (char c : word) {
             if (ref->children.find(c) == ref->children.end()) {
@@ -576,7 +576,7 @@ struct llama_trie {
                             end = lookahead_index;
                             skip = lookahead_index;
                         }
-                        
+
                         auto looktrie_pointer_it = looktrie_pointer->children.find(next_char);
                         while (looktrie_pointer_it != looktrie_pointer->children.end()) {
                             looktrie_pointer = looktrie_pointer_it->second;
@@ -586,7 +586,7 @@ struct llama_trie {
                                 end = lookahead_index;
                                 skip = lookahead_index;
                             }
-                            
+
                             if (lookahead_index == text.size()) {
                                 // End of string
                                 break;
@@ -595,13 +595,13 @@ struct llama_trie {
                             looktrie_pointer_it = looktrie_pointer->children.find(next_char);
                         }
                     }
-                    
+
                     offsets.push_back(start);
                     offsets.push_back(end);
                     reset = true;
                     break;
-                } 
-                
+                }
+
                 auto trie_pointer_it = trie_pointer->children.find(current_char);
                 if (trie_pointer_it != trie_pointer->children.end()) {
                     // The current character being looked at has a match within the trie
@@ -615,20 +615,20 @@ struct llama_trie {
                     state = states.erase(state);
                 }
             }
-            
+
             if (reset) {
                 // Clear the full start (we found a real match)
                 states.clear();
             }
-            
+
             // If this character is a starting character within the trie
             // start keeping track of this partial match.
             auto children_it = root_->children.find(current_char);
             if (current >= skip && children_it != root_->children.end()) {
                 states[current] = children_it->second;
             }
         }
-        
+
         // We have a cut at the end with states.
         for (const auto & state : states) {
             int start = state.first;
@@ -642,7 +642,7 @@ struct llama_trie {
                 break;
             }
         }
-        
+
         offsets.push_back(text.size());
         return offsets;
     }

llama.cpp

@@ -1844,7 +1844,7 @@ struct llama_tokenizer {
 
     void tokenize(const char * text, size_t len, std::vector<llama_vocab::id> & output) {
         symbols_.clear();
-        
+
         // split string into utf8 chars
         int index = 0;
         size_t offs = 0;