update the on device delegate compatibility (#4060)

Summary:
Pull Request resolved: #4060

As discussed offline, mainly two changes in this diff
- add a unit test in the demo backend that will throw `DelegateInvalidCompatibility` when getting a wrong version
- Add doc block for the compatibility error report during init

Reviewed By: mergennachin, kirklandsign

Differential Revision: D58986515

fbshipit-source-id: 0b1da8a770fe564485ceb3a28c3d13831ee37528

Author: cccclai

exir/backend/test/TARGETS

@@ -318,3 +318,22 @@ python_unittest(
         "//executorch/exir/backend/canonical_partitioners:duplicate_constant_node_pass",
     ],
 )
+
+python_unittest(
+    name = "test_compatibility",
+    srcs = [
+        "test_compatibility.py",
+    ],
+    preload_deps = [
+        "//executorch/runtime/executor/test:test_backend_compiler_lib",
+    ],
+    deps = [
+        ":backend_with_compiler_demo",
+        "//caffe2:torch",
+        "//executorch/exir:lib",
+        "//executorch/exir/_serialize:lib",
+        "//executorch/exir/backend:backend_api",
+        "//executorch/exir/backend:compile_spec_schema",
+        "//executorch/extension/pybindings:portable_lib",  # @manual
+    ],
+)

exir/backend/test/backend_with_compiler_demo.py

@@ -82,6 +82,7 @@ def preprocess(
     ) -> PreprocessResult:
         processed_bytes = ""
         number_of_instruction = 0
+        version = "0"
         debug_handle_map = {}
         match_ops = [
             exir_ops.edge.aten.sin.default,
@@ -129,7 +130,12 @@ def preprocess(
             debug_handle_map[new_debug_id] = (original_debug_id,)
         return PreprocessResult(
             processed_bytes=bytes(
-                str(number_of_instruction) + "#" + processed_bytes, encoding="utf8"
+                str(number_of_instruction)
+                + "version:"
+                + version
+                + "#"
+                + processed_bytes,
+                encoding="utf8",
             ),
             debug_handle_map=debug_handle_map,
         )

exir/backend/test/test_backends.py

@@ -194,7 +194,7 @@ def forward(self, x):
             program=program,
             delegate=program.execution_plan[0].delegates[0],
             expected_id=BackendWithCompilerDemo.__name__,
-            expected_processed=b"1#op:demo::aten.sin.default, numel:1, dtype:torch.float32<debug_handle>1#",
+            expected_processed=b"1version:0#op:demo::aten.sin.default, numel:1, dtype:torch.float32<debug_handle>1#",
         )
 
         # Check the delegate instruction
@@ -414,7 +414,7 @@ def forward(self, x):
             program=program,
             delegate=program.execution_plan[0].delegates[0],
             expected_id=BackendWithCompilerDemo.__name__,
-            expected_processed=b"1#op:demo::aten.sin.default, numel:1, dtype:torch.float32<debug_handle>1#",
+            expected_processed=b"1version:0#op:demo::aten.sin.default, numel:1, dtype:torch.float32<debug_handle>1#",
         )
 
         # Check the delegate instruction

exir/backend/test/test_backends_lifted.py

@@ -210,7 +210,7 @@ def forward(self, x):
             program=program,
             delegate=program.execution_plan[0].delegates[0],
             expected_id=BackendWithCompilerDemo.__name__,
-            expected_processed=b"1#op:demo::aten.sin.default, numel:1, dtype:torch.float32<debug_handle>1#",
+            expected_processed=b"1version:0#op:demo::aten.sin.default, numel:1, dtype:torch.float32<debug_handle>1#",
         )
 
         # Check the delegate instruction
@@ -414,7 +414,7 @@ def forward(self, x):
             program=program,
             delegate=program.execution_plan[0].delegates[0],
             expected_id=BackendWithCompilerDemo.__name__,
-            expected_processed=b"1#op:demo::aten.sin.default, numel:1, dtype:torch.float32<debug_handle>1#",
+            expected_processed=b"1version:0#op:demo::aten.sin.default, numel:1, dtype:torch.float32<debug_handle>1#",
         )
 
         # Check the delegate instruction

exir/backend/test/test_compatibility.py

@@ -0,0 +1,67 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import unittest
+
+import torch
+from executorch.exir import to_edge
+from executorch.exir._serialize import _serialize_pte_binary
+from executorch.exir.backend.backend_api import to_backend
+from executorch.exir.backend.compile_spec_schema import CompileSpec
+from executorch.exir.backend.test.backend_with_compiler_demo import (
+    BackendWithCompilerDemo,
+)
+
+from executorch.extension.pybindings.portable_lib import (
+    _load_for_executorch_from_buffer,  # @manual
+)
+from torch.export import export
+
+
+class TestCompatibility(unittest.TestCase):
+    def test_compatibility_in_runtime(self):
+        class SinModule(torch.nn.Module):
+            def __init__(self):
+                super().__init__()
+
+            def forward(self, x):
+                return torch.sin(x)
+
+        sin_module = SinModule()
+        model_inputs = (torch.ones(1),)
+        edgeir_m = to_edge(export(sin_module, model_inputs))
+        max_value = model_inputs[0].shape[0]
+        compile_specs = [CompileSpec("max_value", bytes([max_value]))]
+        lowered_sin_module = to_backend(
+            BackendWithCompilerDemo.__name__, edgeir_m.exported_program(), compile_specs
+        )
+        buff = lowered_sin_module.buffer()
+
+        # The demo backend works well
+        executorch_module = _load_for_executorch_from_buffer(buff)
+        model_inputs = torch.ones(1)
+        _ = executorch_module.forward([model_inputs])
+
+        prog = lowered_sin_module.program()
+        # Rewrite the delegate version number from 0 to 1.
+        prog.backend_delegate_data[0].data = bytes(
+            "1version:1#op:demo::aten.sin.default, numel:1, dtype:torch.float32<debug_handle>1#",
+            encoding="utf8",
+        )
+
+        # Generate the .pte file with the wrong version.
+        buff = bytes(
+            _serialize_pte_binary(
+                program=prog,
+            )
+        )
+
+        # Throw runtime error with error code 0x30, meaning delegate is incompatible.
+        with self.assertRaisesRegex(
+            RuntimeError,
+            "loading method forward failed with error 0x30",
+        ):
+            executorch_module = _load_for_executorch_from_buffer(buff)

pytest.ini

@@ -41,6 +41,7 @@ addopts =
     --ignore=exir/backend/test/demos
     --ignore=exir/backend/test/test_backends.py
     --ignore=exir/backend/test/test_backends_lifted.py
+    --ignore=exir/backend/test/test_compatibility.py
     --ignore=exir/backend/test/test_lowered_backend_module.py
     --ignore=exir/backend/test/test_partitioner.py
     --ignore=exir/tests/test_common.py

runtime/backend/interface.h

@@ -72,7 +72,11 @@ class PyTorchBackendInterface {
    *     implemented by the delegate. This handle is passed to `execute()` and
    *     `destroy()`, and the memory it points to is owned by the backend.
    *     Typically points to a backend-private class/struct.
-   * @returns On error, a value other than Error:Ok.
+   * @returns On error, returns an error code other than Error::Ok. If the
+   *     compiled unit (the preprocessed result from ahead of time) is not
+   *     compatible with the current backend runtime, return the error code
+   *     Error::DelegateInvalidCompatibility. Other backend delegate
+   *     specific error codes can be found in error.h.
    */
   __ET_NODISCARD virtual Result<DelegateHandle*> init(
       BackendInitContext& context,

runtime/executor/test/test_backend_compiler_lib.cpp

@@ -109,9 +109,25 @@ class BackendWithCompiler final : public PyTorchBackendInterface {
     const char* kSignLiteral = "#";
     // The first number is the number of total instruction
     const char* start = static_cast<const char*>(processed->data());
-    char* instruction_number_end =
+
+    const char* kVersion = "version:";
+    const long int kRuntimeVersion = 0;
+    char* version_start =
+        const_cast<char*>(strstr(start, kVersion)) + strlen(kVersion);
+    char* version_end;
+    char* instruction_set_start =
         const_cast<char*>(strstr(start, kSignLiteral));
+
+    long int version = strtol(version_start, &version_end, 10);
+    ET_CHECK_OR_RETURN_ERROR(
+        version == kRuntimeVersion,
+        DelegateInvalidCompatibility,
+        "The version of BackendWithCompiler runtime is %ld, but received an incompatible version %ld instead.",
+        kRuntimeVersion,
+        version);
+    char* instruction_number_end;
     long int instruction_number = strtol(start, &instruction_number_end, 10);
+
     ET_CHECK_OR_RETURN_ERROR(
         instruction_number >= 0,
         InvalidArgument,
@@ -124,7 +140,7 @@ class BackendWithCompiler final : public PyTorchBackendInterface {
         runtime_allocator, DemoOp, instruction_number);
     op_list->numops = static_cast<size_t>(instruction_number);
 
-    parse_delegate(instruction_number_end + 1, kSignLiteral, op_list->ops);
+    parse_delegate(instruction_set_start + 1, kSignLiteral, op_list->ops);
 
     // Can't call `processed->Free()` because op_list points into it.