Merge pull request #490 from msullivan/fix_vec_append

Move the implementation of vec_append from llvm assembly to a regular upcall

Author: pcwalton

src/comp/back/abi.rs

@@ -88,8 +88,8 @@ fn bzero_glue_name() -> str {
     ret "rust_bzero_glue";
 }
 
-fn vec_append_glue_name() -> str {
-    ret "rust_vec_append_glue";
+fn yield_glue_name() -> str {
+    ret "rust_yield_glue";
 }
 
 fn no_op_type_glue_name() -> str {

src/comp/back/upcall.rs

@@ -5,6 +5,7 @@ import trans::decl_cdecl_fn;
 import trans::T_f32;
 import trans::T_f64;
 import trans::T_fn;
+import trans::T_bool;
 import trans::T_i8;
 import trans::T_i32;
 import trans::T_int;
@@ -53,6 +54,7 @@ type upcalls = rec(
     ValueRef new_str,
     ValueRef new_vec,
     ValueRef vec_grow,
+    ValueRef vec_append,
     ValueRef get_type_desc,
     ValueRef new_task,
     ValueRef start_task,
@@ -111,6 +113,9 @@ fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls {
         vec_grow=d("vec_grow", [T_opaque_vec_ptr(), T_size_t(),
                                    T_ptr(T_int()), T_ptr(T_tydesc(tn))],
                    T_opaque_vec_ptr()),
+        vec_append=d("vec_append", [T_ptr(T_tydesc(tn)), T_ptr(T_tydesc(tn)),
+                                    T_ptr(T_opaque_vec_ptr()),
+                                    T_opaque_vec_ptr(), T_bool()], T_void()),
         get_type_desc=d("get_type_desc",
                         [T_ptr(T_nil()), T_size_t(), T_size_t(),
                          T_size_t(), T_ptr(T_ptr(T_tydesc(tn)))],

src/comp/middle/trans.rs

@@ -85,8 +85,7 @@ state obj namegen(mutable int i) {
 
 type derived_tydesc_info = rec(ValueRef lltydesc, bool escapes);
 
-type glue_fns = rec(ValueRef no_op_type_glue,
-                    ValueRef vec_append_glue);
+type glue_fns = rec(ValueRef no_op_type_glue);
 
 type tydesc_info = rec(ty::t ty,
                        ValueRef tydesc,
@@ -1972,7 +1971,7 @@ fn declare_generic_glue(&@local_ctxt cx,
         fn_nm = mangle_internal_name_by_seq(cx.ccx,
                                             "glue_" + name);
     }
-    auto llfn = decl_fastcall_fn(cx.ccx.llmod, fn_nm, llfnty);
+    auto llfn = decl_cdecl_fn(cx.ccx.llmod, fn_nm, llfnty);
     set_glue_inlining(cx, llfn, t);
     ret llfn;
 }
@@ -3263,11 +3262,11 @@ fn call_tydesc_glue_full(&@block_ctxt cx, ValueRef v,
     auto llfnptr = cx.build.GEP(tydesc, [C_int(0), C_int(field)]);
     auto llfn = cx.build.Load(llfnptr);
 
-    cx.build.FastCall(llfn, [C_null(T_ptr(T_nil())),
-                                cx.fcx.lltaskptr,
-                                C_null(T_ptr(T_nil())),
-                                lltydescs,
-                                llrawptr]);
+    cx.build.Call(llfn, [C_null(T_ptr(T_nil())),
+                         cx.fcx.lltaskptr,
+                         C_null(T_ptr(T_nil())),
+                         lltydescs,
+                         llrawptr]);
 }
 
 fn call_tydesc_glue(&@block_ctxt cx, ValueRef v,
@@ -3342,7 +3341,7 @@ fn call_cmp_glue(&@block_ctxt cx,
                                    llrawrhsptr,
                                    llop];
 
-    r.bcx.build.FastCall(llfn, llargs);
+    r.bcx.build.Call(llfn, llargs);
 
     ret res(r.bcx, r.bcx.build.Load(llcmpresultptr));
 }
@@ -3711,7 +3710,7 @@ fn trans_vec_append(&@block_ctxt cx, &ty::t t,
     auto dst = bcx.build.PointerCast(lhs, T_ptr(T_opaque_vec_ptr()));
     auto src = bcx.build.PointerCast(rhs, T_opaque_vec_ptr());
 
-    ret res(bcx, bcx.build.FastCall(cx.fcx.lcx.ccx.glues.vec_append_glue,
+    ret res(bcx, bcx.build.Call(cx.fcx.lcx.ccx.upcalls.vec_append,
                                     [cx.fcx.lltaskptr,
                                         llvec_tydesc.val,
                                         llelt_tydesc.val,
@@ -8742,39 +8741,6 @@ fn make_no_op_type_glue(ValueRef fun) {
     new_builder(llbb).RetVoid();
 }
 
-fn make_vec_append_glue(ModuleRef llmod, type_names tn) -> ValueRef {
-    /*
-     * Args to vec_append_glue:
-     *
-     *   0. (Implicit) task ptr
-     *
-     *   1. Pointer to the tydesc of the vec, so that we can tell if it's gc
-     *      mem, and have a tydesc to pass to malloc if we're allocating anew.
-     *
-     *   2. Pointer to the tydesc of the vec's stored element type, so that
-     *      elements can be copied to a newly alloc'ed vec if one must be
-     *      created.
-     *
-     *   3. Dst vec ptr (i.e. ptr to ptr to rust_vec).
-     *
-     *   4. Src vec (i.e. ptr to rust_vec).
-     *
-     *   5. Flag indicating whether to skip trailing null on dst.
-     *
-     */
-
-    auto ty = T_fn([T_taskptr(tn),
-                       T_ptr(T_tydesc(tn)),
-                       T_ptr(T_tydesc(tn)),
-                       T_ptr(T_opaque_vec_ptr()),
-                       T_opaque_vec_ptr(), T_bool()],
-                   T_void());
-
-    auto llfn = decl_fastcall_fn(llmod, abi::vec_append_glue_name(), ty);
-    ret llfn;
-}
-
-
 fn vec_fill(&@block_ctxt bcx, ValueRef v) -> ValueRef {
     ret bcx.build.Load(bcx.build.GEP(v, [C_int(0),
                                             C_int(abi::vec_elt_fill)]));
@@ -8787,8 +8753,7 @@ fn vec_p0(&@block_ctxt bcx, ValueRef v) -> ValueRef {
 }
 
 fn make_glues(ModuleRef llmod, &type_names tn) -> @glue_fns {
-    ret @rec(no_op_type_glue = decl_no_op_type_glue(llmod, tn),
-             vec_append_glue = make_vec_append_glue(llmod, tn));
+    ret @rec(no_op_type_glue = decl_no_op_type_glue(llmod, tn));
 }
 
 fn make_common_glue(&session::session sess, &str output) {

src/rt/rust_internal.h

@@ -206,19 +206,27 @@ struct rust_timer {
 
 #include "rust_util.h"
 
+typedef void CDECL (glue_fn)(void *, rust_task *, void *,
+                             const type_desc **, void *);
+typedef void CDECL (cmp_glue_fn)(void *, rust_task *, void *,
+                                 const type_desc **,
+                                 void *, void *, int8_t);
+
+
 struct type_desc {
     // First part of type_desc is known to compiler.
     // first_param = &descs[1] if dynamic, null if static.
     const type_desc **first_param;
     size_t size;
     size_t align;
-    uintptr_t copy_glue_off;
-    uintptr_t drop_glue_off;
-    uintptr_t free_glue_off;
-    uintptr_t sever_glue_off;    // For GC.
-    uintptr_t mark_glue_off;     // For GC.
-    uintptr_t obj_drop_glue_off; // For custom destructors.
+    glue_fn *take_glue;
+    glue_fn *drop_glue;
+    glue_fn *free_glue;
+    glue_fn *sever_glue;    // For GC.
+    glue_fn *mark_glue;     // For GC.
+    glue_fn *obj_drop_glue; // For custom destructors.
     uintptr_t is_stateful;
+    cmp_glue_fn *cmp_glue;
 
     // Residual fields past here are known only to runtime.
     UT_hash_handle hh;

src/rt/rust_upcall.cpp

@@ -415,7 +415,7 @@ upcall_vec_grow(rust_task *task,
          *
          * Step 3 is a bit tricky.  We don't know how to properly copy the
          * elements in the runtime (all we have are bits in a buffer; no
-         * type infromation and no copy glue).  What we do instead is set the
+         * type information and no copy glue).  What we do instead is set the
          * need_copy outparam flag to indicate to our caller (vec-copy glue)
          * that we need the copies performed for us.
          */
@@ -436,6 +436,53 @@ upcall_vec_grow(rust_task *task,
     return v;
 }
 
+// Copy elements from one vector to another,
+// dealing with reference counts
+static inline void
+copy_elements(rust_task *task, type_desc *elem_t,
+              void *pdst, void *psrc, size_t n)
+{
+    char *dst = (char *)pdst, *src = (char *)psrc;
+
+    // increment the refcount of each element of the vector
+    if (elem_t->take_glue) {
+        glue_fn *take_glue = elem_t->take_glue;
+        size_t elem_size = elem_t->size;
+        const type_desc **tydescs = elem_t->first_param;
+        for (char *p = src; p < src+n; p += elem_size) {
+            take_glue(NULL, task, NULL, tydescs, p);
+        }
+    }
+    memmove(dst, src, n);
+}
+
+extern "C" CDECL void
+upcall_vec_append(rust_task *task, type_desc *t, type_desc *elem_t,
+                  rust_vec **dst_ptr, rust_vec *src, bool skip_null)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_vec *dst = *dst_ptr;
+    uintptr_t need_copy;
+    size_t n_src_bytes = skip_null ? src->fill - 1 : src->fill;
+    size_t n_dst_bytes = skip_null ? dst->fill - 1 : dst->fill;
+    rust_vec *new_vec = upcall_vec_grow(task, dst, n_src_bytes,
+                                        &need_copy, t);
+
+    if (need_copy) {
+        // Copy any dst elements in, omitting null if doing str.
+        copy_elements(task, elem_t, &new_vec->data, &dst->data, n_dst_bytes);
+    }
+
+    // Copy any src elements in, carrying along null if doing str.
+    void *new_end = (void *)((char *)new_vec->data + n_dst_bytes);
+    copy_elements(task, elem_t, new_end, &src->data, src->fill);
+    new_vec->fill = n_dst_bytes + src->fill;
+
+    // Write new_vec back through the alias we were given.
+    *dst_ptr = new_vec;
+}
+
+
 extern "C" CDECL type_desc *
 upcall_get_type_desc(rust_task *task,
                      void *curr_crate, // ignored, legacy compat.

src/rt/rustrt.def.in

@@ -73,6 +73,7 @@ upcall_start_thread
 upcall_trace_str
 upcall_trace_word
 upcall_vec_grow
+upcall_vec_append
 upcall_yield
 vec_alloc
 vec_alloc_mut