Skip to content

Rewrite etdump debug data value comparision #4152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Rewrite etdump debug data value comparision #4152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions sdk/inspector/_inspector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
gen_graphs_from_etrecord,
inflate_runtime_output,
is_debug_output,
is_inference_output_equal,
ProgramOutput,
RESERVED_FRAMEWORK_EVENT_NAMES,
TIME_SCALE_DICT,
Expand Down Expand Up @@ -571,8 +572,10 @@ def _populate_debugging_related_fields(
debug_data = [debug_event.debug_entry for debug_event in debug_events]
else:
for debug_event, value in zip(debug_events, debug_data):
assert (
debug_event.debug_entry == value
v1 = inflate_runtime_output(debug_event.debug_entry, output_buffer)
v2 = inflate_runtime_output(value, output_buffer)
assert is_inference_output_equal(
v1, v2
), """Corresponding debug events in multiple iterations of the model
must have the same debug entry values. This is not the case for the
intermediate data present in this ETDump and indicates potential issues
Expand Down
14 changes: 14 additions & 0 deletions sdk/inspector/_inspector_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,20 @@ class TimeScale(Enum):
ProgramOutput: TypeAlias = List[InferenceOutput]


# Compare whether two InferenceOutputs are equal
def is_inference_output_equal(
output1: InferenceOutput, output2: InferenceOutput
) -> bool:
if isinstance(output1, torch.Tensor) and isinstance(output2, torch.Tensor):
return torch.equal(output1, output2)
elif isinstance(output1, List) and isinstance(output2, List):
return all(torch.equal(t1, t2) for t1, t2 in zip(output1, output2))
elif output1 == output2:
return True
else:
return False


# Given a ETDump Tensor object and offset, extract into a torch.Tensor
def _parse_tensor_value(
tensor: Optional[Tensor], output_buffer: Optional[bytes]
Expand Down
128 changes: 128 additions & 0 deletions sdk/inspector/tests/inspector_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@

from executorch.sdk.inspector import _inspector, Event, EventBlock, Inspector, PerfData
from executorch.sdk.inspector._inspector import (
DebugEventSignature,
flatcc,
InstructionEvent,
InstructionEventSignature,
ProfileEventSignature,
Expand Down Expand Up @@ -273,6 +275,132 @@ def test_inspector_get_exported_program(self):
)
)

def test_populate_debugging_related_fields_raises_for_inconsistent_events(self):
ret_event: Event = Event(
name="event",
)

debug_event_0 = flatcc.DebugEvent(
chain_index=1,
instruction_id=0,
debug_entry=flatcc.Value(
val=flatcc.ValueType.TENSOR.value,
tensor=flatcc.Tensor(
scalar_type=flatcc.ScalarType.INT,
sizes=[2],
strides=[1],
offset=12345,
),
tensor_list=None,
int_value=None,
float_value=None,
double_value=None,
bool_value=None,
output=None,
),
)

# Note the sizes of this tensor are different from the previous one
debug_event_1 = flatcc.DebugEvent(
chain_index=1,
instruction_id=0,
debug_entry=flatcc.Value(
val=flatcc.ValueType.TENSOR.value,
tensor=flatcc.Tensor(
scalar_type=flatcc.ScalarType.INT,
sizes=[1],
strides=[1],
offset=23456,
),
tensor_list=None,
int_value=None,
float_value=None,
double_value=None,
bool_value=None,
output=None,
),
)

instruction_event_0 = InstructionEvent(
signature=InstructionEventSignature(1, 1), debug_events=[debug_event_0]
)
instruction_event_1 = InstructionEvent(
signature=InstructionEventSignature(1, 1), debug_events=[debug_event_1]
)

events = [instruction_event_0, instruction_event_1]

# Expect AssertionError because 2 tensors have different sizes
with self.assertRaises(AssertionError):
Event._populate_debugging_related_fields(
ret_event=ret_event,
debug_event_signature=DebugEventSignature(instruction_id=1),
events=events,
)

def test_populate_debugging_related_fields_passes_for_consistent_events(self):
ret_event: Event = Event(
name="event",
)

debug_event_0 = flatcc.DebugEvent(
chain_index=1,
instruction_id=0,
debug_entry=flatcc.Value(
val=flatcc.ValueType.TENSOR.value,
tensor=flatcc.Tensor(
scalar_type=flatcc.ScalarType.INT,
sizes=[1],
strides=[1],
offset=12345,
),
tensor_list=None,
int_value=None,
float_value=None,
double_value=None,
bool_value=None,
output=None,
),
)

# Same as the event above except for offset
debug_event_1 = flatcc.DebugEvent(
chain_index=1,
instruction_id=0,
debug_entry=flatcc.Value(
val=flatcc.ValueType.TENSOR.value,
tensor=flatcc.Tensor(
scalar_type=flatcc.ScalarType.INT,
sizes=[1],
strides=[1],
offset=23456,
),
tensor_list=None,
int_value=None,
float_value=None,
double_value=None,
bool_value=None,
output=None,
),
)

instruction_event_0 = InstructionEvent(
signature=InstructionEventSignature(1, 1), debug_events=[debug_event_0]
)
instruction_event_1 = InstructionEvent(
signature=InstructionEventSignature(1, 1), debug_events=[debug_event_1]
)

events = [instruction_event_0, instruction_event_1]

with patch.object(_inspector, "is_inference_output_equal", return_value=True):
# Expect it runs with no error because is_inference_output_equal() is mocked to return True
Event._populate_debugging_related_fields(
ret_event=ret_event,
debug_event_signature=DebugEventSignature(instruction_id=1),
events=events,
)

def _gen_random_float_list(self) -> List[float]:
return [random.uniform(0, 10) for _ in range(RAW_DATA_SIZE)]

Expand Down
42 changes: 42 additions & 0 deletions sdk/inspector/tests/inspector_utils_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import unittest
from typing import Dict, Tuple

import torch

from executorch.sdk import generate_etrecord, parse_etrecord

from executorch.sdk.debug_format.base_schema import (
Expand All @@ -25,6 +27,7 @@
EDGE_DIALECT_GRAPH_KEY,
find_populated_event,
gen_graphs_from_etrecord,
is_inference_output_equal,
)


Expand Down Expand Up @@ -126,6 +129,45 @@ def test_find_populated_event(self):
)
self.assertEqual(find_populated_event(event), profile_event)

def test_is_inference_output_equal_returns_false_for_different_tensor_values(self):
self.assertFalse(
is_inference_output_equal(
torch.tensor([[2, 1], [4, 3]]),
torch.tensor([[5, 6], [7, 8]]),
)
)

def test_is_inference_output_equal_returns_false_for_different_tensor_lists(self):
tensor_list_1 = (
[
torch.tensor([[1, 2], [3, 4]]),
torch.tensor([[1, 2], [3, 4]]),
torch.tensor([[1, 2], [3, 4]]),
],
)
tensor_list_2 = [
torch.tensor([[1, 2], [3, 4]]),
torch.tensor([[1, 2], [3, 4]]),
]
# Not equal because of different number of tensors
self.assertFalse(is_inference_output_equal(tensor_list_1, tensor_list_2))

def test_is_inference_output_equal_returns_true_for_same_tensor_values(self):
self.assertTrue(
is_inference_output_equal(
torch.tensor([[2, 1], [4, 3]]),
torch.tensor([[2, 1], [4, 3]]),
)
)

def test_is_inference_output_equal_returns_true_for_same_strs(self):
self.assertTrue(
is_inference_output_equal(
"value_string",
"value_string",
)
)


def gen_mock_operator_graph_with_expected_map() -> (
Tuple[OperatorGraph, Dict[int, OperatorNode]]
Expand Down
Loading