Do not error out when op is not in ATen namespace

exir/program/test/test_program.py

@@ -537,16 +537,18 @@ def _test_edge_dialect_verifier(self, callable, validate_ir=True):
         _ = to_edge(exported_foo, compile_config=edge_compile_config)
 
     def test_edge_dialect_custom_op(self):
+        # We shouldn't error out if there's a custom op in the graph.
         def _use_foo_add(a: torch.Tensor, b: torch.Tensor):
             return torch.ops.exir_program_test_op.foo(a, b)
 
         from torch._export.verifier import SpecViolationError
 
-        with self.assertRaises(SpecViolationError):
+        try:
+            # This should not raise error
             self._test_edge_dialect_verifier(_use_foo_add)
-
-        # This should not raise error
-        self._test_edge_dialect_verifier(_use_foo_add, False)
+            self._test_edge_dialect_verifier(_use_foo_add, False)
+        except SpecViolationError:
+            self.fail("Should not error out on custom op")
 
     def _test_model_with_non_decomp_partitioner(self, model: torch.nn.Module):
         # This is the pre-dispatch export that we will be switching to primarily

exir/verification/verifier.py

@@ -98,17 +98,7 @@ def _get_exception_list(self) -> List[torch._ops.OpOverload]:
     def check_valid_op(self, op):
         if isinstance(op, OpOverload):
             # TODO These special ops should be removable easily.
-            if (
-                op.namespace
-                in [
-                    "quantized_decomposed",
-                    "boltnn_nimble",
-                    "nimble",
-                    "quantized",
-                    "dim_order_ops",
-                ]
-                or op in self._get_exception_list()
-            ):
+            if op.namespace != "aten" or op in self._get_exception_list():
                 return
             if torch.Tag.core not in op.tags and torch.Tag.view_copy not in op.tags:
                 # NOTE(qihan): whether view_copy operators are marked as canonical is still under