|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# All rights reserved. |
| 3 | +# |
| 4 | +# This source code is licensed under the BSD-style license found in the |
| 5 | +# LICENSE file in the root directory of this source tree. |
| 6 | + |
| 7 | +# pyre-unsafe |
| 8 | + |
| 9 | +import json |
| 10 | +from typing import Any, Dict |
| 11 | + |
| 12 | +import torch |
| 13 | +from executorch.examples.models.checkpoint import ( |
| 14 | + get_checkpoint_dtype, |
| 15 | + get_default_model_resource_dir, |
| 16 | +) |
| 17 | + |
| 18 | +from executorch.examples.models.model_base import EagerModelBase |
| 19 | +from torchtune.models.llama3_2_vision._component_builders import llama3_2_vision_decoder |
| 20 | +from torchtune.models.llama3_2_vision._convert_weights import llama3_vision_meta_to_tune |
| 21 | + |
| 22 | + |
| 23 | +def to_decoder_checkpoint(checkpoint: Dict[str, Any]) -> Dict[str, Any]: |
| 24 | + """ |
| 25 | + Extracts and formats the decoder-related weights from the checkpoint. The checkpoint contains |
| 26 | + weight names prefixed with "encoder"/"decoder", such as "encoder.layer.etc" or "decoder.norm.scale". |
| 27 | + To load the text decoder on its own, the "decoder" prefix needs to be removed. |
| 28 | + """ |
| 29 | + return { |
| 30 | + ".".join(weight.split(".")[1:]): value |
| 31 | + for weight, value in checkpoint.items() |
| 32 | + if weight.startswith("decoder") |
| 33 | + } |
| 34 | + |
| 35 | + |
| 36 | +class Llama3_2Decoder(EagerModelBase): |
| 37 | + """ |
| 38 | + Just the text decoder portions of the Llama3.2 multimodal model. |
| 39 | + """ |
| 40 | + |
| 41 | + def __init__(self, **kwargs): |
| 42 | + # Set member vars from kwargs. |
| 43 | + self.max_seq_len = kwargs.get( |
| 44 | + "max_seq_len", 8192 |
| 45 | + ) # Trained to be a lot larger, but this value is kept small because of static kv cache at the moment. |
| 46 | + self.encoder_max_seq_len = kwargs.get( |
| 47 | + "encoder_max_seq_len", int(4 * (448 / 14) ** 2 + 1) |
| 48 | + ) # Same as above. |
| 49 | + self.generate_full_logits = kwargs.get("generate_full_logits", False) |
| 50 | + self.enable_dynamic_shape = kwargs.get("enable_dynamic_shape", False) |
| 51 | + self.output_prune_map_path = kwargs.get("output_prune_map_path", None) |
| 52 | + self.use_kv_cache = kwargs.get("use_kv_cache", False) |
| 53 | + self.verbose = kwargs.get("verbose", False) |
| 54 | + self.args = kwargs.get("args", None) |
| 55 | + |
| 56 | + ckpt_dir = get_default_model_resource_dir(__file__) |
| 57 | + # Single checkpoint file. |
| 58 | + checkpoint_path = kwargs.get("checkpoint", ckpt_dir / "demo_rand_params.pth") |
| 59 | + # Sharded checkpoint. |
| 60 | + checkpoint_dir = kwargs.get("checkpoint_dir", None) |
| 61 | + params_path = kwargs.get("params", ckpt_dir / "demo_config.json") |
| 62 | + |
| 63 | + self.causal_mask = torch.tril( |
| 64 | + torch.ones( |
| 65 | + size=(self.max_seq_len, self.max_seq_len), |
| 66 | + dtype=torch.bool, |
| 67 | + ) |
| 68 | + ) |
| 69 | + self.input_pos = torch.arange(self.max_seq_len) |
| 70 | + |
| 71 | + # Load checkpoint and params. |
| 72 | + device = "cpu" |
| 73 | + if checkpoint_dir is not None: |
| 74 | + raise NotImplementedError( |
| 75 | + "Sharded checkpoint not yet supported for Llama3_2Decoder." |
| 76 | + ) |
| 77 | + else: |
| 78 | + checkpoint = torch.load( |
| 79 | + checkpoint_path, map_location=device, weights_only=False, mmap=True |
| 80 | + ) |
| 81 | + checkpoint = llama3_vision_meta_to_tune(checkpoint) |
| 82 | + checkpoint = to_decoder_checkpoint(checkpoint) |
| 83 | + with open(params_path, "r") as f: |
| 84 | + params = json.loads(f.read()) |
| 85 | + |
| 86 | + # Find dtype from checkpoint. (skip for now) |
| 87 | + self.dtype = get_checkpoint_dtype(checkpoint) |
| 88 | + |
| 89 | + # Load model. |
| 90 | + # Cannot use "with torch.device("meta"):" because it causes some exceptions during export, |
| 91 | + # i.e. the model isn't fully initialized or something. |
| 92 | + self.model_ = llama3_2_vision_decoder( |
| 93 | + vocab_size=params["vocab_size"], |
| 94 | + num_layers=params["n_layers"], |
| 95 | + fusion_interval=params["fusion_interval"], |
| 96 | + num_special_tokens=params["n_special_tokens"], |
| 97 | + num_heads=params["n_heads"], |
| 98 | + num_kv_heads=params["n_kv_heads"], |
| 99 | + embed_dim=params["dim"], |
| 100 | + max_seq_len=self.max_seq_len, |
| 101 | + encoder_max_seq_len=self.encoder_max_seq_len, |
| 102 | + rope_base=params["rope_theta"], |
| 103 | + intermediate_dim=params["intermediate_dim"], |
| 104 | + ) |
| 105 | + # Save params for future use. |
| 106 | + for param_name, param_val in params.items(): |
| 107 | + setattr(self.model_, param_name, param_val) |
| 108 | + |
| 109 | + # Quantize. (skip for now) |
| 110 | + |
| 111 | + # Load checkpoint. |
| 112 | + missing, unexpected = self.model_.load_state_dict( |
| 113 | + checkpoint, |
| 114 | + strict=False, |
| 115 | + assign=True, |
| 116 | + ) |
| 117 | + if kwargs.get("verbose", False): |
| 118 | + print("============= missing keys ================") |
| 119 | + print(missing) |
| 120 | + print("============= /missing ================") |
| 121 | + print("============= unexpected keys ================") |
| 122 | + print(unexpected) |
| 123 | + print("============= /unexpected ================") |
| 124 | + |
| 125 | + # Prune the output layer if output_prune_map is provided. |
| 126 | + output_prune_map = None |
| 127 | + if self.output_prune_map_path is not None: |
| 128 | + from executorch.examples.models.llama2.source_transformation.prune_output import ( |
| 129 | + prune_output_vocab, |
| 130 | + ) |
| 131 | + |
| 132 | + with open(self.output_prune_map_path, "r") as f: |
| 133 | + output_prune_map = json.load(f) |
| 134 | + # Change keys from string to int (json only supports string keys) |
| 135 | + output_prune_map = {int(k): v for (k, v) in output_prune_map.items()} |
| 136 | + |
| 137 | + self.model_ = prune_output_vocab(self.model_, output_prune_map) |
| 138 | + |
| 139 | + # if self.use_kv_cache: |
| 140 | + # print("Setting up KV cache on the model...") |
| 141 | + # self.model_.setup_caches( |
| 142 | + # batch_size=1, |
| 143 | + # dtype=self.dtype, |
| 144 | + # ) |
| 145 | + |
| 146 | + def get_eager_model(self) -> torch.nn.Module: |
| 147 | + if self.dtype: |
| 148 | + return self.model_.to(self.dtype) |
| 149 | + else: |
| 150 | + return self.model_.to(torch.float16) |
| 151 | + |
| 152 | + def get_example_inputs(self): |
| 153 | + return (torch.ones(1, 64, dtype=torch.long),) |
| 154 | + |
| 155 | + def get_example_kwarg_inputs(self): |
| 156 | + # TODO: add input_pos and mask when after making cache work. |
| 157 | + return { |
| 158 | + # "mask": self.causal_mask[None, 64, None, :], |
| 159 | + # "encoder_input": None, |
| 160 | + # "encoder_mask": None, |
| 161 | + # "input_pos": self.input_pos[None, 64] |
| 162 | + } |
| 163 | + |
| 164 | + def get_dynamic_shapes(self): |
| 165 | + batch_size = 1 |
| 166 | + dim_seq_len = torch.export.Dim("token_dim", min=1, max=self.max_seq_len) |
| 167 | + dynamic_shapes = { |
| 168 | + "tokens": {0: batch_size, 1: dim_seq_len}, |
| 169 | + # "encoder_input": {0: 1, 1: dim_enc, 2: 4096}, |
| 170 | + # "encoder_mask": {0: 1, 1: dim, 2: dim_enc}, |
| 171 | + # "mask": {0: batch_size, 1: dim_seq_len, 2: self.max_seq_len}, |
| 172 | + # "input_pos" : {0: batch_size, 1: dim_seq_len}, |
| 173 | + } |
| 174 | + return dynamic_shapes |
0 commit comments