Add eval for eager et (#2856)

Summary: Pull Request resolved: #2856

Test Plan:
Imported from GitHub, without a `Test Plan:` line.
```
python3 -m examples.models.llama2.eval_llama --pte stories_fp32.pte -t ../llama-models/stories/tokenizer.model
```
Results:
```
100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [2:21:53<00:00, 1702.63s/it]
wikitext: {'word_perplexity,none': 10885.21588720899, 'word_perplexity_stderr,none': 'N/A', 'byte_perplexity,none': 6.144013580095854, 'byte_perplexity_stderr,none': 'N/A', 'bits_per_byte,none': 2.619181404847365, 'bits_per_byte_stderr,none': 'N/A', 'alias': 'wikitext'}
```

Reviewed By: Jack-Khuu

Differential Revision: D55777043

Pulled By: lucylq

fbshipit-source-id: d3d52bc5fd7674d69c2bf36a084974f3276d32a7

Author: lucylq

examples/models/llama2/eval_llama_lib.py

@@ -9,8 +9,8 @@
 from typing import Optional
 
 import lm_eval
-
 import torch
+
 from lm_eval.api.model import LM
 from lm_eval.evaluator import evaluate
 from lm_eval.models.huggingface import HFLM as eval_wrapper
@@ -33,7 +33,7 @@ class GPTFastEvalWrapper(eval_wrapper):
     def __init__(
         self,
         model: nn.Module,
-        tokenizer,
+        tokenizer: SentencePieceProcessor,
         max_seq_length: Optional[int] = None,
     ):
         super().__init__()
@@ -97,16 +97,18 @@ def __init__(
         max_seq_length: Optional[int] = None,
     ):
         super().__init__(None, tokenizer, max_seq_length)
-        self._model = model
+        self._model = model  # Expects model to be path to a .pte file
 
-    def _model_call(self, inps):
-        # Given inps (tokens), return the logits from a single
-        # forward call
+        from executorch.extension.pybindings.portable_lib import _load_for_executorch
 
-        # Example:
-        # inps: Tensor of shape (1, N)
-        # logits: Tensor of shape (1, N, 32000)
-        pass
+        self._et_model = _load_for_executorch(self._model)
+
+    def _model_call(self, inps):
+        # Given inps (tokens), return the logits from a single forward call
+        # inps: Tensor of shape (1, max_seq_len - 1)
+        # logits: Tensor of shape (1, max_seq_len - 1, 32000)
+        result = self._et_model.forward((inps,))
+        return result[0]
 
 
 class ETRunnerEvalWrapper(GPTFastEvalWrapper):
@@ -198,7 +200,9 @@ def gen_eval_wrapper(
         return ETEagerEvalWrapper(
             model=model,
             tokenizer=tokenizer,
-            max_seq_length=args.max_seq_length,
+            # Exported model takes at most (max_seq_length - 1) tokens.
+            # Note that the eager model takes at most max_seq_length tokens.
+            max_seq_length=args.max_seq_length - 1,
         )
 
     # GPTFastEvalWrapper: Create a wrapper around a pre-exported model