Remove llama related stuff out of bpe_tokenizer

Pull Request resolved: #4235

We don't need to initialize `vocab_`, `vocab_scores_`, etc. They will be initialized anyway while loading the tokenizer binary. A benefit of removing them is that we can remove these llama related default values and make `bpe_tokenizer` agnostic to models.
ghstack-source-id: 233578697

Differential Revision: [D59664556](https://our.internmc.facebook.com/intern/diff/D59664556/)

Author: helunwencser

examples/models/llama2/tokenizer/bpe_tokenizer.cpp

@@ -24,12 +24,6 @@ static int compare_tokens(const void* a, const void* b) {
 }
 
 BPETokenizer::BPETokenizer() : Tokenizer() {
-  vocab_size_ = kDefaultVocabSize;
-  vocab_ = std::make_unique<char*[]>(kDefaultVocabSize);
-  vocab_scores_ = std::make_unique<float[]>(kDefaultVocabSize);
-  sorted_vocab_ = std::make_unique<TokenIndex[]>(kDefaultVocabSize);
-  bos_tok_ = kDefaultBosTokenId;
-  eos_tok_ = kDefaultEosTokenId;
   for (int i = 0; i < 256; i++) {
     byte_pieces_[i * 2] = (unsigned char)i;
     byte_pieces_[i * 2 + 1] = '\0';

examples/models/llama2/tokenizer/bpe_tokenizer.h

@@ -14,11 +14,6 @@
 namespace torch {
 namespace executor {
 
-// Default values for llama2
-constexpr int32_t kDefaultVocabSize = 32000;
-constexpr uint64_t kDefaultBosTokenId = 1;
-constexpr uint64_t kDefaultEosTokenId = 2;
-
 struct TokenIndex {
   const char* str;
   int32_t id;