File tree Expand file tree Collapse file tree 1 file changed +15
-1
lines changed Expand file tree Collapse file tree 1 file changed +15
-1
lines changed Original file line number Diff line number Diff line change @@ -111,16 +111,30 @@ def use_et_backend() -> bool:
111
111
##########################################################################
112
112
### set and get target precision for this model ###
113
113
114
- precision = torch . float32
114
+ precision = None
115
115
116
116
117
117
def set_precision (dtype ):
118
+ """set_precision() is a torchchat-internal API that records the dtype we're building the model for.
119
+ The precision is recorded for future queries by get_precision(), so that when building a model,
120
+ or performing optimizations, we can query the type the user is building the model for.
121
+ This is an informational value that can be used when we want to know what type to build for (e.g., a kv cache).
122
+ Changing the `precision` does not change the precision of the model.
123
+ """
124
+
118
125
global precision
126
+ assert precision is None , "only set precision once to avoid inconsistent answers during different phases of model build and export"
119
127
precision = dtype
120
128
121
129
122
130
def get_precision ():
131
+ """get_precision() is a torchchat-internal API that returns the dtype we're building the model for, as specified by the `--dtype` CLI option+,
132
+ or the precision quantizer.
133
+ """
123
134
global precision
135
+ # if (and only if) precision has not been set, update it to the default value torch.float32
136
+ if precision is None :
137
+ precision = torch .float32
124
138
return precision
125
139
126
140
You can’t perform that action at this time.
0 commit comments