[BE] Delay imports (#377)

Do not import neither sentencepiece nor tiktoken until it is requested

Same with HF hub

Author: malfet

build/builder.py

@@ -18,9 +18,6 @@
 from config.model_config import resolve_model_config
 from quantize import quantize_model
 
-from sentencepiece import SentencePieceProcessor
-from tokenizer.tiktoken import Tokenizer as TiktokenTokenizer
-
 from build.model import Transformer
 from build.utils import device_sync, name_to_dtype
 
@@ -201,8 +198,10 @@ def from_args(cls, args):  # -> TokenizerArgs:
 
 def _initialize_tokenizer(tokenizer_args: TokenizerArgs):
     if tokenizer_args.is_sentencepiece:
+        from sentencepiece import SentencePieceProcessor
         return SentencePieceProcessor(model_file=str(tokenizer_args.tokenizer_path))
     elif tokenizer_args.is_tiktoken:
+        from tokenizer.tiktoken import Tokenizer as TiktokenTokenizer
         return TiktokenTokenizer(model_path=str(tokenizer_args.tokenizer_path))
     else:
         raise RuntimeError("must specify a valid tokenizer in TokenizerArgs")

download.py

@@ -16,13 +16,13 @@
     resolve_model_config,
 )
 
-from requests.exceptions import HTTPError
 
 
 def _download_hf_snapshot(
     model_config: ModelConfig, artifact_dir: Path, hf_token: Optional[str]
 ):
     from huggingface_hub import snapshot_download
+    from requests.exceptions import HTTPError
 
     # Download and store the HF model artifacts.
     print(f"Downloading {model_config.name} from HuggingFace...")