[mypyc] Support var arg in CallC, replace new_dict_op (#8948)

Related: mypyc/mypyc#709, mypyc/mypyc#734

Summary:

* introduce variable arguments in CallC
* replace old new_dict_op (which relies on a specialized emit callback) 
  with two CallC ops: dict_new_op and dict_build_op, which handles create 
  an empty dict and a dict from k-v pairs.
* fix related IR tests, especially separate the testDel case into three 
  subcases which test list, dict and attributes respectively.

Author: TH3CHARLie

mypyc/ir/ops.py

@@ -1151,13 +1151,15 @@ def __init__(self,
                  ret_type: RType,
                  steals: StealsDescription,
                  error_kind: int,
-                 line: int) -> None:
+                 line: int,
+                 var_arg_idx: int = -1) -> None:
         self.error_kind = error_kind
         super().__init__(line)
         self.function_name = function_name
         self.args = args
         self.type = ret_type
         self.steals = steals
+        self.var_arg_idx = var_arg_idx  # the position of the first variable argument in args
 
     def to_str(self, env: Environment) -> str:
         args_str = ', '.join(env.format('%r', arg) for arg in self.args)

mypyc/irbuild/classdef.py

@@ -21,7 +21,7 @@
     dataclass_sleight_of_hand, pytype_from_template_op, py_calc_meta_op, type_object_op,
     not_implemented_op, true_op
 )
-from mypyc.primitives.dict_ops import dict_set_item_op, new_dict_op
+from mypyc.primitives.dict_ops import dict_set_item_op, dict_new_op
 from mypyc.primitives.tuple_ops import new_tuple_op
 from mypyc.common import SELF_NAME
 from mypyc.irbuild.util import (
@@ -73,7 +73,7 @@ def transform_class_def(builder: IRBuilder, cdef: ClassDef) -> None:
         # We populate __annotations__ for non-extension classes
         # because dataclasses uses it to determine which attributes to compute on.
         # TODO: Maybe generate more precise types for annotations
-        non_ext_anns = builder.primitive_op(new_dict_op, [], cdef.line)
+        non_ext_anns = builder.call_c(dict_new_op, [], cdef.line)
         non_ext = NonExtClassInfo(non_ext_dict, non_ext_bases, non_ext_anns, non_ext_metaclass)
         dataclass_non_ext = None
         type_obj = None
@@ -258,7 +258,7 @@ def setup_non_ext_dict(builder: IRBuilder,
     builder.goto(exit_block)
 
     builder.activate_block(false_block)
-    builder.assign(non_ext_dict, builder.primitive_op(new_dict_op, [], cdef.line), cdef.line)
+    builder.assign(non_ext_dict, builder.call_c(dict_new_op, [], cdef.line), cdef.line)
     builder.goto(exit_block)
     builder.activate_block(exit_block)
 
@@ -518,9 +518,9 @@ def dataclass_non_ext_info(builder: IRBuilder, cdef: ClassDef) -> Optional[NonEx
     """
     if is_dataclass(cdef):
         return NonExtClassInfo(
-            builder.primitive_op(new_dict_op, [], cdef.line),
+            builder.call_c(dict_new_op, [], cdef.line),
             builder.add(TupleSet([], cdef.line)),
-            builder.primitive_op(new_dict_op, [], cdef.line),
+            builder.call_c(dict_new_op, [], cdef.line),
             builder.primitive_op(type_object_op, [], cdef.line),
         )
     else:

mypyc/irbuild/expression.py

@@ -25,7 +25,7 @@
 from mypyc.primitives.misc_ops import new_slice_op, ellipsis_op, type_op
 from mypyc.primitives.list_ops import new_list_op, list_append_op, list_extend_op
 from mypyc.primitives.tuple_ops import list_tuple_op
-from mypyc.primitives.dict_ops import new_dict_op, dict_set_item_op
+from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op
 from mypyc.primitives.set_ops import new_set_op, set_add_op, set_update_op
 from mypyc.irbuild.specialize import specializers
 from mypyc.irbuild.builder import IRBuilder
@@ -541,7 +541,7 @@ def gen_inner_stmts() -> None:
 
 
 def transform_dictionary_comprehension(builder: IRBuilder, o: DictionaryComprehension) -> Value:
-    d = builder.primitive_op(new_dict_op, [], o.line)
+    d = builder.call_c(dict_new_op, [], o.line)
     loop_params = list(zip(o.indices, o.sequences, o.condlists))
 
     def gen_inner_stmts() -> None:

mypyc/irbuild/ll_builder.py

@@ -26,6 +26,7 @@
 from mypyc.ir.rtypes import (
     RType, RUnion, RInstance, optional_value_type, int_rprimitive, float_rprimitive,
     bool_rprimitive, list_rprimitive, str_rprimitive, is_none_rprimitive, object_rprimitive,
+    c_int_rprimitive
 )
 from mypyc.ir.func_ir import FuncDecl, FuncSignature
 from mypyc.ir.class_ir import ClassIR, all_concrete_classes
@@ -41,7 +42,9 @@
     list_extend_op, list_len_op, new_list_op
 )
 from mypyc.primitives.tuple_ops import list_tuple_op, new_tuple_op
-from mypyc.primitives.dict_ops import new_dict_op, dict_update_in_display_op
+from mypyc.primitives.dict_ops import (
+    dict_update_in_display_op, dict_new_op, dict_build_op
+)
 from mypyc.primitives.generic_ops import (
     py_getattr_op, py_call_op, py_call_with_kwargs_op, py_method_call_op
 )
@@ -566,12 +569,14 @@ def unary_op(self,
 
     def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value:
         result = None  # type: Union[Value, None]
-        initial_items = []  # type: List[Value]
+        keys = []  # type: List[Value]
+        values = []  # type: List[Value]
         for key, value in key_value_pairs:
             if key is not None:
                 # key:value
                 if result is None:
-                    initial_items.extend((key, value))
+                    keys.append(key)
+                    values.append(value)
                     continue
 
                 self.translate_special_method_call(
@@ -583,7 +588,7 @@ def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value:
             else:
                 # **value
                 if result is None:
-                    result = self.primitive_op(new_dict_op, initial_items, line)
+                    result = self._create_dict(keys, values, line)
 
                 self.primitive_op(
                     dict_update_in_display_op,
@@ -592,7 +597,7 @@ def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value:
                 )
 
         if result is None:
-            result = self.primitive_op(new_dict_op, initial_items, line)
+            result = self._create_dict(keys, values, line)
 
         return result
 
@@ -685,12 +690,22 @@ def call_c(self,
                result_type: Optional[RType] = None) -> Value:
         # handle void function via singleton RVoid instance
         coerced = []
-        for i, arg in enumerate(args):
+        # coerce fixed number arguments
+        for i in range(min(len(args), len(desc.arg_types))):
             formal_type = desc.arg_types[i]
+            arg = args[i]
             arg = self.coerce(arg, formal_type, line)
             coerced.append(arg)
+        # coerce any var_arg
+        var_arg_idx = -1
+        if desc.var_arg_type is not None:
+            var_arg_idx = len(desc.arg_types)
+            for i in range(len(desc.arg_types), len(args)):
+                arg = args[i]
+                arg = self.coerce(arg, desc.var_arg_type, line)
+                coerced.append(arg)
         target = self.add(CallC(desc.c_function_name, coerced, desc.return_type, desc.steals,
-                                desc.error_kind, line))
+                                desc.error_kind, line, var_arg_idx))
         if result_type and not is_runtime_subtype(target.type, result_type):
             if is_none_rprimitive(result_type):
                 # Special case None return. The actual result may actually be a bool
@@ -859,3 +874,18 @@ def translate_eq_cmp(self,
             ltype,
             line
         )
+
+    def _create_dict(self,
+                     keys: List[Value],
+                     values: List[Value],
+                     line: int) -> Value:
+        """Create a dictionary(possibly empty) using keys and values"""
+        # keys and values should have the same number of items
+        size = len(keys)
+        if size > 0:
+            load_size_op = self.add(LoadInt(size, -1, c_int_rprimitive))
+            # merge keys and values
+            items = [i for t in list(zip(keys, values)) for i in t]
+            return self.call_c(dict_build_op, [load_size_op] + items, line)
+        else:
+            return self.call_c(dict_new_op, [], line)

mypyc/primitives/dict_ops.py

@@ -5,13 +5,13 @@
 from mypyc.ir.ops import EmitterInterface, ERR_FALSE, ERR_MAGIC, ERR_NEVER
 from mypyc.ir.rtypes import (
     dict_rprimitive, object_rprimitive, bool_rprimitive, int_rprimitive,
-    list_rprimitive, dict_next_rtuple_single, dict_next_rtuple_pair
+    list_rprimitive, dict_next_rtuple_single, dict_next_rtuple_pair, c_int_rprimitive
 )
 
 from mypyc.primitives.registry import (
     name_ref_op, method_op, binary_op, func_op, custom_op,
     simple_emit, negative_int_emit, call_emit, call_negative_bool_emit,
-    name_emit,
+    name_emit, c_custom_op
 )
 
 
@@ -88,25 +88,22 @@
     error_kind=ERR_MAGIC,
     emit=simple_emit('{dest} = CPyDict_Get({args[0]}, {args[1]}, Py_None);'))
 
-
-def emit_new_dict(emitter: EmitterInterface, args: List[str], dest: str) -> None:
-    if not args:
-        emitter.emit_line('%s = PyDict_New();' % (dest,))
-        return
-
-    emitter.emit_line('%s = CPyDict_Build(%s, %s);' % (dest, len(args) // 2, ', '.join(args)))
-
+# Construct an empty dictionary.
+dict_new_op = c_custom_op(
+    arg_types=[],
+    return_type=dict_rprimitive,
+    c_function_name='PyDict_New',
+    error_kind=ERR_MAGIC)
 
 # Construct a dictionary from keys and values.
-# Arguments are (key1, value1, ..., keyN, valueN).
-new_dict_op = custom_op(
-    name='builtins.dict',
-    arg_types=[object_rprimitive],
-    is_var_arg=True,
-    result_type=dict_rprimitive,
-    format_str='{dest} = {{{colon_args}}}',
+# Positional argument is the number of key-value pairs
+# Variable arguments are (key1, value1, ..., keyN, valueN).
+dict_build_op = c_custom_op(
+    arg_types=[c_int_rprimitive],
+    return_type=dict_rprimitive,
+    c_function_name='CPyDict_Build',
     error_kind=ERR_MAGIC,
-    emit=emit_new_dict)
+    var_arg_type=object_rprimitive,)
 
 # Construct a dictionary from another dictionary.
 func_op(

mypyc/primitives/registry.py

@@ -46,6 +46,7 @@
     'CFunctionDescription',  [('name', str),
                               ('arg_types', List[RType]),
                               ('return_type', RType),
+                              ('var_arg_type', Optional[RType]),
                               ('c_function_name', str),
                               ('error_kind', int),
                               ('steals', StealsDescription),
@@ -337,10 +338,25 @@ def c_method_op(name: str,
                 return_type: RType,
                 c_function_name: str,
                 error_kind: int,
+                var_arg_type: Optional[RType] = None,
                 steals: StealsDescription = False,
                 priority: int = 1) -> CFunctionDescription:
+    """Define a c function call op that replaces a method call.
+
+    This will be automatically generated by matching against the AST.
+
+    Args:
+        name: short name of the method (for example, 'append')
+        arg_types: argument types; the receiver is always the first argument
+        return_type: type of the return value. Use void_rtype to represent void.
+        c_function_name: name of the C function to call
+        error_kind: how errors are represented in the result (one of ERR_*)
+        var_arg_type: type of all variable arguments
+        steals: description of arguments that this steals (ref count wise)
+        priority: if multiple ops match, the one with the highest priority is picked
+    """
     ops = c_method_call_ops.setdefault(name, [])
-    desc = CFunctionDescription(name, arg_types, return_type,
+    desc = CFunctionDescription(name, arg_types, return_type, var_arg_type,
                                 c_function_name, error_kind, steals, priority)
     ops.append(desc)
     return desc
@@ -351,10 +367,21 @@ def c_function_op(name: str,
                   return_type: RType,
                   c_function_name: str,
                   error_kind: int,
+                  var_arg_type: Optional[RType] = None,
                   steals: StealsDescription = False,
                   priority: int = 1) -> CFunctionDescription:
+    """Define a c function call op that replaces a function call.
+
+    This will be automatically generated by matching against the AST.
+
+    Most arguments are similar to c_method_op().
+
+    Args:
+        name: full name of the function
+        arg_types: positional argument types for which this applies
+    """
     ops = c_function_ops.setdefault(name, [])
-    desc = CFunctionDescription(name, arg_types, return_type,
+    desc = CFunctionDescription(name, arg_types, return_type, var_arg_type,
                                 c_function_name, error_kind, steals, priority)
     ops.append(desc)
     return desc
@@ -365,28 +392,58 @@ def c_binary_op(name: str,
                 return_type: RType,
                 c_function_name: str,
                 error_kind: int,
+                var_arg_type: Optional[RType] = None,
                 steals: StealsDescription = False,
                 priority: int = 1) -> CFunctionDescription:
+    """Define a c function call op for a binary operation.
+
+    This will be automatically generated by matching against the AST.
+
+    Most arguments are similar to c_method_op(), but exactly two argument types
+    are expected.
+    """
     ops = c_binary_ops.setdefault(name, [])
-    desc = CFunctionDescription(name, arg_types, return_type,
+    desc = CFunctionDescription(name, arg_types, return_type, var_arg_type,
                             c_function_name, error_kind, steals, priority)
     ops.append(desc)
     return desc
 
 
+def c_custom_op(arg_types: List[RType],
+                return_type: RType,
+                c_function_name: str,
+                error_kind: int,
+                var_arg_type: Optional[RType] = None,
+                steals: StealsDescription = False) -> CFunctionDescription:
+    """Create a one-off CallC op that can't be automatically generated from the AST.
+
+    Most arguments are similar to c_method_op().
+    """
+    return CFunctionDescription('<custom>', arg_types, return_type, var_arg_type,
+                            c_function_name, error_kind, steals, 0)
+
+
 def c_unary_op(name: str,
                arg_type: RType,
                return_type: RType,
                c_function_name: str,
                error_kind: int,
                steals: StealsDescription = False,
                priority: int = 1) -> CFunctionDescription:
+    """Define a c function call op for an unary operation.
+
+    This will be automatically generated by matching against the AST.
+
+    Most arguments are similar to c_method_op(), but exactly one argument type
+    is expected.
+    """
     ops = c_unary_ops.setdefault(name, [])
-    desc = CFunctionDescription(name, [arg_type], return_type,
+    desc = CFunctionDescription(name, [arg_type], return_type, None,
                             c_function_name, error_kind, steals, priority)
     ops.append(desc)
     return desc
 
+
 # Import various modules that set up global state.
 import mypyc.primitives.int_ops  # noqa
 import mypyc.primitives.str_ops  # noqa