@@ -545,7 +545,7 @@ typedef llama_buffer llama_ctx_buffer;
545
545
546
546
struct llama_trie_node {
547
547
llama_trie_node (): is_terminator(false ) {}
548
-
548
+
549
549
std::unordered_map<char , llama_trie_node*> children;
550
550
bool is_terminator;
551
551
};
@@ -560,7 +560,7 @@ struct llama_trie {
560
560
if (word.empty ()) {
561
561
return ;
562
562
}
563
-
563
+
564
564
llama_trie_node *ref = root_;
565
565
for (char c : word) {
566
566
if (ref->children .find (c) == ref->children .end ()) {
@@ -630,7 +630,7 @@ struct llama_trie {
630
630
end = lookahead_index;
631
631
skip = lookahead_index;
632
632
}
633
-
633
+
634
634
auto looktrie_pointer_it = looktrie_pointer->children .find (next_char);
635
635
while (looktrie_pointer_it != looktrie_pointer->children .end ()) {
636
636
looktrie_pointer = looktrie_pointer_it->second ;
@@ -640,7 +640,7 @@ struct llama_trie {
640
640
end = lookahead_index;
641
641
skip = lookahead_index;
642
642
}
643
-
643
+
644
644
if (lookahead_index == text.size ()) {
645
645
// End of string
646
646
break ;
@@ -649,13 +649,13 @@ struct llama_trie {
649
649
looktrie_pointer_it = looktrie_pointer->children .find (next_char);
650
650
}
651
651
}
652
-
652
+
653
653
offsets.push_back (start);
654
654
offsets.push_back (end);
655
655
reset = true ;
656
656
break ;
657
- }
658
-
657
+ }
658
+
659
659
auto trie_pointer_it = trie_pointer->children .find (current_char);
660
660
if (trie_pointer_it != trie_pointer->children .end ()) {
661
661
// The current character being looked at has a match within the trie
@@ -669,20 +669,20 @@ struct llama_trie {
669
669
state = states.erase (state);
670
670
}
671
671
}
672
-
672
+
673
673
if (reset) {
674
674
// Clear the full start (we found a real match)
675
675
states.clear ();
676
676
}
677
-
677
+
678
678
// If this character is a starting character within the trie
679
679
// start keeping track of this partial match.
680
680
auto children_it = root_->children .find (current_char);
681
681
if (current >= skip && children_it != root_->children .end ()) {
682
682
states[current] = children_it->second ;
683
683
}
684
684
}
685
-
685
+
686
686
// We have a cut at the end with states.
687
687
for (const auto & state : states) {
688
688
int start = state.first ;
@@ -696,7 +696,7 @@ struct llama_trie {
696
696
break ;
697
697
}
698
698
}
699
-
699
+
700
700
offsets.push_back (text.size ());
701
701
return offsets;
702
702
}
0 commit comments