bug in sycning amax history

Author: drisspg

float8_experimental/float8_linear_utils.py

@@ -164,6 +164,24 @@ def get_float8_layers(model: torch.nn.Module):
     return fp8_layers
 
 
+def get_float8_layers_dtype(model: torch.nn.Module):
+    """Iterates through the model and returns all the Float8Linear layers.
+    Args:
+        model (torch.nn.Module): The model to look for Float8Linear layers in.
+    """
+    fp8_dtype_fw = set()
+    fp8_dtype_bw = set()
+    # Get all fp8 layers and tensors
+    for child in model.modules():
+        if isinstance(child, Float8Linear):
+            fp8_dtype_fw.add(child.fp8_dtype_fw)
+            fp8_dtype_bw.add(child.fp8_dtype_bw)
+
+    assert len(fp8_dtype_fw) == 1, "All fp8 layers must have the same fp8_dtype_fw"
+    assert len(fp8_dtype_bw) == 1, "All fp8 layers must have the same fp8_dtype_bw"
+    return fp8_dtype_fw.pop(), fp8_dtype_bw.pop()
+
+
 @torch.no_grad()
 def sync_float8_amax_and_scale_history(model: torch.nn.Module, fp8_layers=None) -> None:
     """
@@ -197,6 +215,8 @@ def sync_float8_amax_and_scale_history(model: torch.nn.Module, fp8_layers=None)
         )
         return
 
+    fp8_dtype_fw, fp8_dtype_bw = get_float8_layers_dtype(model)
+
     def inner_func():
         """Why do we have this inner_function?
 
@@ -293,13 +313,13 @@ def inner_func():
 
         # Calculate the new scales from the updated history stacks
         new_x_scales = amax_history_to_scale_stack(
-            fp8_x_amax_history_stack, torch.float8_e4m3fn, x_dtype, scale_fn_recipe
+            fp8_x_amax_history_stack, fp8_dtype_fw, x_dtype, scale_fn_recipe
         )
         new_w_scales = amax_history_to_scale_stack(
-            fp8_w_amax_history_stack, torch.float8_e4m3fn, x_dtype, scale_fn_recipe
+            fp8_w_amax_history_stack, fp8_dtype_fw, x_dtype, scale_fn_recipe
         )
         new_dL_dY_scales = amax_history_to_scale_stack(
-            fp8_dL_dY_amax_history_stack, torch.float8_e5m2, x_dtype, scale_fn_recipe
+            fp8_dL_dY_amax_history_stack, fp8_dtype_bw, x_dtype, scale_fn_recipe
         )
 
         # Iterate through the layers and update the scales

test/test_base.py

@@ -69,6 +69,7 @@ def _test_linear_impl(
         m_fp8 = get_float8_linear(
             linear_type, m_ref, emulate, use_activation_hooks, fp8_dtypes
         )
+        y_ref, y_fp8 = None, None
         for _ in range(2):
             if linear_requires_sync(linear_type):
                 sync_float8_amax_and_scale_history(m_fp8)
@@ -77,7 +78,7 @@ def _test_linear_impl(
             y_ref = m_ref(x)
             y_ref.sum().backward()
 
-        assert y_ref.shape == y_fp8.shape
+            assert y_ref.shape == y_fp8.shape
 
         y_sqnr = compute_error(y_ref, y_fp8)
         g_sqnr = compute_error(m_ref.weight.grad, m_fp8.weight.grad)
@@ -131,10 +132,10 @@ def _test_linear_impl(
             # verify initialization flags got updated
             assert m_fp8.is_amax_initialized, "Amax was not properly initialized"
 
-    @pytest.mark.parametrize("emulate", [True, False] if is_H100 else [True])
+    @pytest.mark.parametrize("emulate", [True] if is_H100 else [True])
     @pytest.mark.parametrize("x_shape", [(16, 16), (2, 16, 16), (3, 2, 16, 16)])
-    @pytest.mark.parametrize("linear_type", [LinearType.DELAYED, LinearType.DYNAMIC])
-    @pytest.mark.parametrize("use_activation_hooks", [True, False])
+    @pytest.mark.parametrize("linear_type", [LinearType.DYNAMIC, LinearType.DELAYED])
+    @pytest.mark.parametrize("use_activation_hooks", [False])
     @pytest.mark.usefixtures("x_fail_activation_hooks_with_delayed")
     @unittest.skipIf(not torch.cuda.is_available(), "CUDA not available")
     def test_linear_nobias(