Update on "Move tokenizer.py into extension/llm/tokenizer"

as title

Differential Revision: [D59698943](https://our.internmc.facebook.com/intern/diff/D59698943/)

[ghstack-poisoned]

Author: helunwencser

.ci/scripts/test_llama.sh

@@ -171,7 +171,7 @@ $PYTHON_EXECUTABLE -m examples.models.llama2.export_llama ${EXPORT_ARGS}
 
 # Create tokenizer.bin.
 echo "Creating tokenizer.bin"
-$PYTHON_EXECUTABLE -m examples.models.llama2.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
+$PYTHON_EXECUTABLE -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
 
 
 RUNTIME_ARGS="--model_path=${EXPORTED_MODEL_NAME} --tokenizer_path=tokenizer.bin --prompt=Once --temperature=0 --seq_len=10"

backends/vulkan/docs/android_demo.md

@@ -65,7 +65,7 @@ echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps":
 Then, create a tokenizer binary file:
 
 ```shell
-python -m examples.models.llama2.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
+python -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
 ```
 
 Finally, export the `stories110M.pt` file into an ExecuTorch program:

examples/models/llama2/README.md

@@ -90,7 +90,7 @@ You can export and run the original Llama 2 7B model.
 4. Create tokenizer.bin.
 
     ```
-    python -m examples.models.llama2.tokenizer.tokenizer -t <tokenizer.model> -o tokenizer.bin
+    python -m extension.llm.tokenizer.tokenizer -t <tokenizer.model> -o tokenizer.bin
     ```
 
 ### Option B: Download and export stories110M model
@@ -113,7 +113,7 @@ If you want to deploy and run a smaller model for educational purposes. From `ex
 4. Create tokenizer.bin.
 
     ```
-    python -m examples.models.llama2.tokenizer.tokenizer -t <tokenizer.model> -o tokenizer.bin
+    python -m extension.llm.tokenizer.tokenizer -t <tokenizer.model> -o tokenizer.bin
     ```
 
 ### Option C: Download and export Llama 3 8B instruct model

examples/models/llama2/eval_llama_lib.py

@@ -15,11 +15,11 @@
     get_quantizer_and_quant_params,
 )
 from executorch.examples.models.llama2.tokenizer.tiktoken import Tokenizer as Tiktoken
-from executorch.examples.models.llama2.tokenizer.tokenizer import (
-    Tokenizer as SentencePieceTokenizer,
-)
 
 from executorch.extension.llm.export import LLMEdgeManager
+from executorch.extension.llm.tokenizer.tokenizer import (
+    Tokenizer as SentencePieceTokenizer,
+)
 
 from lm_eval.api.model import LM
 

examples/models/llama2/evaluate/eager_eval.py

@@ -10,7 +10,7 @@
 import lm_eval
 import torch
 from executorch.examples.models.llama2.tokenizer.tiktoken import Tokenizer as Tiktoken
-from executorch.examples.models.llama2.tokenizer.tokenizer import (
+from executorch.extension.llm.tokenizer.tokenizer import (
     Tokenizer as SentencePieceTokenizer,
 )
 

examples/qualcomm/llama2/README.md

@@ -23,7 +23,7 @@ wget "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt"
 wget "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model"
 
 # tokenizer.bin:
-python -m examples.models.llama2.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
+python -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
 
 # params.json:
 echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json

extension/llm/tokenizer/targets.bzl

@@ -1,6 +1,11 @@
 load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
 
 def define_common_targets():
+    """Defines targets that should be shared between fbcode and xplat.
+
+    The directory containing this targets.bzl file should also contain both
+    TARGETS and BUCK files that call this function.
+    """
     runtime.python_library(
         name = "tokenizer_py_lib",
         srcs = [
@@ -15,6 +20,7 @@ def define_common_targets():
             "//bento_kernels/...",
         ],
         _is_external_target = True,
+        # TODO: Define an external_deps entry for sentencepiece instead of pointing to an fbsource path.
         deps = [] if runtime.is_oss else ["fbsource//third-party/pypi/sentencepiece:sentencepiece"],
     )