Add implementation of emscripten_memcpy_big based on bulk memory.

These new functions live in `libbulkmemory` which only gets included
if bulk memory is enabled (either via `-mbulk-memory` directly or
indirectly via `-pthread).

benchmark results for benchmark.test_memcpy_1mb:

```
 v8:                       mean: 1.666
 v8-bulkmemory:            mean: 1.598
 v8-standalone-bulkmemory: mean: 1.576
 v8-standalone:            mean: 3.197
```

Here we can see the that when bulk memory is enabled its at least as
fast if not faster than the JS version.

v8-standalone doesn't have emscripten_memcpy_big at all is is much
slower, as expected.  By adding `-mbulk-memory` the standalone version
becomes just as fast as the non-standalone.

Author: sbc100

emcc.py

@@ -1603,6 +1603,7 @@ def phase_setup(options, state, newargs):
     if '-matomics' not in newargs:
       newargs += ['-matomics']
     if '-mbulk-memory' not in newargs:
+      settings.BULK_MEMORY = 1
       newargs += ['-mbulk-memory']
 
   if 'DISABLE_EXCEPTION_CATCHING' in user_settings and 'EXCEPTION_CATCHING_ALLOWED' in user_settings:
@@ -2440,6 +2441,8 @@ def phase_linker_setup(options, state, newargs):
     settings.JS_LIBRARIES.append((0, shared.path_from_root('src', 'library_wasm_worker.js')))
 
   settings.SUPPORTS_GLOBALTHIS = feature_matrix.caniuse(feature_matrix.Feature.GLOBALTHIS)
+  if not settings.BULK_MEMORY:
+    settings.BULK_MEMORY = feature_matrix.caniuse(feature_matrix.Feature.BULK_MEMORY)
 
   if settings.AUDIO_WORKLET:
     if not settings.SUPPORTS_GLOBALTHIS:
@@ -3571,6 +3574,8 @@ def consume_arg_file():
       settings.DISABLE_EXCEPTION_CATCHING = 1
       settings.DISABLE_EXCEPTION_THROWING = 1
       settings.WASM_EXCEPTIONS = 0
+    elif arg == '-mbulk-memory':
+      settings.BULK_MEMORY = 1
     elif arg == '-fexceptions':
       # TODO Currently -fexceptions only means Emscripten EH. Switch to wasm
       # exception handling by default when -fexceptions is given when wasm

src/library.js

@@ -389,9 +389,11 @@ mergeInto(LibraryManager.library, {
   // variant, so we should never emit emscripten_memcpy_big() in the build.
   // In STANDALONE_WASM we avoid the emscripten_memcpy_big dependency so keep
   // the wasm file standalone.
+  // In BULK_MEMORY mode we include native versions of these functions based
+  // on memory.fill and memory.copy.
   // In MAIN_MODULE=1 or EMCC_FORCE_STDLIBS mode all of libc is force included
   // so we cannot override parts of it, and therefore cannot use libc_optz.
-#if (SHRINK_LEVEL < 2 || LINKABLE || process.env.EMCC_FORCE_STDLIBS) && !STANDALONE_WASM
+#if (SHRINK_LEVEL < 2 || LINKABLE || process.env.EMCC_FORCE_STDLIBS) && !STANDALONE_WASM && !BULK_MEMORY
 
 #if MIN_CHROME_VERSION < 45 || MIN_EDGE_VERSION < 14 || MIN_FIREFOX_VERSION < 34 || MIN_IE_VERSION != TARGET_NOT_SUPPORTED || MIN_SAFARI_VERSION < 100101
   // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/copyWithin lists browsers that support TypedArray.prototype.copyWithin, but it

src/settings_internal.js

@@ -256,3 +256,5 @@ var POST_JS_FILES = [];
 
 // Set when -pthread / -sPTHREADS is passed
 var PTHREADS = false;
+
+var BULK_MEMORY = false;

system/lib/libc/emscripten_internal.h

@@ -30,6 +30,7 @@ extern "C" {
 void emscripten_memcpy_big(void* __restrict__ dest,
                            const void* __restrict__ src,
                            size_t n) EM_IMPORT(emscripten_memcpy_big);
+void emscripten_memset_big(void* ptr, char value, size_t n);
 
 void emscripten_notify_memory_growth(size_t memory_index);
 

system/lib/libc/emscripten_memcpy.c

@@ -29,7 +29,7 @@ static void *__memcpy(void *restrict dest, const void *restrict src, size_t n) {
   unsigned char *block_aligned_d_end;
   unsigned char *d_end;
 
-#ifndef EMSCRIPTEN_STANDALONE_WASM
+#if !defined(EMSCRIPTEN_STANDALONE_WASM) || defined(__wasm_bulk_memory__)
   if (n >= 512) {
     emscripten_memcpy_big(dest, src, n);
     return dest;

system/lib/libc/emscripten_memcpy_big.S

@@ -0,0 +1,14 @@
+#ifdef __wasm64__
+#define PTR i64
+#else
+#define PTR i32
+#endif
+
+.globl emscripten_memcpy_big
+emscripten_memcpy_big:
+  .functype emscripten_memcpy_big (PTR, PTR, PTR) -> ()
+  local.get 0
+  local.get 1
+  local.get 2
+  memory.copy 0, 0
+  end_function

system/lib/libc/emscripten_memset.c

@@ -1,8 +1,9 @@
-// XXX EMSCRIPTEN ASAN: build an uninstrumented version of memset
-#if defined(__EMSCRIPTEN__) && defined(__has_feature)
-#if __has_feature(address_sanitizer)
-#define memset __attribute__((no_sanitize("address"))) emscripten_builtin_memset
-#endif
+#include "emscripten_internal.h"
+
+// build an uninstrumented version of memset
+#if defined(__has_feature) && __has_feature(address_sanitizer)
+__attribute__((no_sanitize("address"))) static void *memset(void *str, int c, size_t n);
+__attribute__((no_sanitize("address"))) void *__musl_memset(void *str, int c, size_t n);
 #endif
 
 #ifdef EMSCRIPTEN_OPTIMIZE_FOR_OZ
@@ -16,6 +17,21 @@ void *memset(void *str, int c, size_t n) {
   return str;
 }
 
+#elif defined(__wasm_bulk_memory__)
+
+__attribute__((__weak__)) void *__musl_memset(void *str, int c, size_t n);
+#define memset __musl_memset
+#include "musl/src/string/memset.c"
+#undef memset
+
+void *memset(void *str, int c, size_t n) {
+  if (n >= 512) {
+    emscripten_memset_big(str, c, n);
+    return str;
+  }
+  return __musl_memset(str, c, n);
+}
+
 #else
 
 #include "musl/src/string/memset.c"

system/lib/libc/emscripten_memset_big.S

@@ -0,0 +1,14 @@
+#ifdef __wasm64__
+#define PTR i64
+#else
+#define PTR i32
+#endif
+
+.globl emscripten_memset_big
+emscripten_memset_big:
+  .functype emscripten_memset_big (PTR, i32, PTR) -> ()
+  local.get 0
+  local.get 1
+  local.get 2
+  memory.fill 0
+  end_function

system/lib/standalone/standalone.c

@@ -152,7 +152,7 @@ int emscripten_resize_heap(size_t size) {
 }
 
 double emscripten_get_now(void) {
-  return (1000 * clock()) / (double)CLOCKS_PER_SEC;
+  return (1000ll * clock()) / (double)CLOCKS_PER_SEC;
 }
 
 // C++ ABI

test/other/test_memops_bulk_memory.c

@@ -0,0 +1,13 @@
+#include <assert.h>
+#include <string.h>
+
+const char *hello = "hello";
+const char *world = "world";
+
+int main() {
+  char buffer[100];
+  memset(buffer, 'a', 100);
+  memcpy(buffer, hello, strlen(hello) + 1);
+  assert(strcmp(buffer, hello) == 0);
+  return 0;
+}

test/test_benchmark.py

@@ -13,6 +13,7 @@
 import zlib
 from pathlib import Path
 from typing import List
+from subprocess import CalledProcessError
 
 if __name__ == '__main__':
   raise Exception('do not run this file directly; do something like: test/runner.py benchmark')
@@ -83,6 +84,7 @@ def bench(self, args, output_parser=None, reps=TEST_REPS, expected_output=None):
         except Exception as e:
           print(str(e))
           print('Parsing benchmark results failed, output was: ' + output)
+          return
       self.times.append(curr)
 
   def display(self, baseline=None):
@@ -158,7 +160,14 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat
     self.filename = final
 
   def run(self, args):
-    return run_process([self.filename] + args, stdout=PIPE, stderr=PIPE, check=False).stdout
+    try:
+      return run_process([self.filename] + args, stdout=PIPE, stderr=PIPE, check=False).stdout
+    except CalledProcessError as e:
+      print("-- STDOUT:")
+      print(e.stdout)
+      print("-- STDERR:")
+      print(e.stderr)
+      raise
 
   def get_output_files(self):
     return [self.filename]
@@ -214,7 +223,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat
     if common.EMTEST_FORCE64:
       cmd += ['--profiling']
     else:
-      cmd += ['--closure=1', '-sMINIMAL_RUNTIME']
+      cmd += ['--closure=1'] #, '-sMINIMAL_RUNTIME']
     # add additional emcc args at the end, which may override other things
     # above, such as minimal runtime
     cmd += emcc_args + self.extra_args
@@ -229,7 +238,14 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat
     self.filename = final
 
   def run(self, args):
-    return jsrun.run_js(self.filename, engine=self.engine, args=args, stderr=PIPE)
+    try:
+      return jsrun.run_js(self.filename, engine=self.engine, args=args, stderr=PIPE)
+    except CalledProcessError as e:
+      print("-- STDOUT:")
+      print(e.stdout)
+      print("-- STDERR:")
+      print(e.stderr)
+      raise
 
   def get_output_files(self):
     ret = [self.filename]
@@ -356,7 +372,7 @@ def get_output_files(self):
 
 if not common.EMTEST_FORCE64:
   benchmarkers += [
-    NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]),
+    #NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]),
     # NativeBenchmarker('gcc',   ['gcc', '-no-pie'],  ['g++', '-no-pie'])
   ]
 
@@ -372,9 +388,17 @@ def get_output_files(self):
     ]
   else:
     benchmarkers += [
-      EmscriptenBenchmarker(default_v8_name, aot_v8),
-      EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']),
-      EmscriptenBenchmarker(default_v8_name + '-ctors', aot_v8, ['-sEVAL_CTORS']),
+      EmscriptenBenchmarker(default_v8_name, aot_v8,
+                            ['-fno-builtin']),
+      EmscriptenBenchmarker(default_v8_name + '-bulkmemory', aot_v8,
+                            ['-mbulk-memory', '-fno-builtin']),
+      EmscriptenBenchmarker(default_v8_name + '-standalone-bulkmemory', aot_v8,
+                            ['-sSTANDALONE_WASM', '-sWASM_BIGINT', '-mbulk-memory', '-fno-builtin']),
+      EmscriptenBenchmarker(default_v8_name + '-standalone', aot_v8,
+                            ['-sSTANDALONE_WASM', '-sWASM_BIGINT', '-fno-builtin']),
+      #EmscriptenBenchmarker(default_v8_name, aot_v8),
+      #EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']),
+      #EmscriptenBenchmarker(default_v8_name + '-ctors', aot_v8, ['-sEVAL_CTORS']),
       # EmscriptenWasm2CBenchmarker('wasm2c')
     ]
   if os.path.exists(CHEERP_BIN):

test/test_other.py

@@ -13412,3 +13412,24 @@ def test_wasi_random_get(self):
   @requires_node
   def test_wasi_sched_yield(self):
     self.run_wasi_test_suite_test('wasi_sched_yield')
+
+  def test_memops_bulk_memory(self):
+    self.emcc_args += ['--profiling-funcs', '-fno-builtin']
+    # By default we expect to find _emscripten_memcpy_big in the generaed JS and not in the
+    # native code.
+    self.do_runf(test_file('other/test_memops_bulk_memory.c'))
+    self.assertContained('_emscripten_memcpy_big', read_file('test_memops_bulk_memory.js'))
+    funcs = self.parse_wasm('test_memops_bulk_memory.wasm')[2]
+    self.assertNotIn('$emscripten_memcpy_big', funcs)
+
+    # With bulk memory enabled we expect *not* to find it.
+    self.do_runf(test_file('other/test_memops_bulk_memory.c'), emcc_args=['-mbulk-memory'])
+    self.assertNotContained('_emscripten_memcpy_big', read_file('test_memops_bulk_memory.js'))
+    funcs = self.parse_wasm('test_memops_bulk_memory.wasm')[2]
+    self.assertIn('$emscripten_memcpy_big', funcs)
+
+    # -pthread implicitly enables bulk memory too.
+    self.do_runf(test_file('other/test_memops_bulk_memory.c'), emcc_args=['-pthread'])
+    self.assertNotContained('_emscripten_memcpy_big', read_file('test_memops_bulk_memory.js'))
+    funcs = self.parse_wasm('test_memops_bulk_memory.wasm')[2]
+    self.assertIn('$emscripten_memcpy_big', funcs)

tools/settings.py

@@ -71,6 +71,7 @@
     'DEFAULT_TO_CXX',
     'WASM_OBJECT_FILES',
     'WASM_WORKERS',
+    'BULK_MEMORY',
 
     # Internal settings used during compilation
     'EXCEPTION_CATCHING_ALLOWED',

tools/system_libs.py

@@ -1274,6 +1274,20 @@ def can_use(self):
         not settings.LINKABLE and not os.environ.get('EMCC_FORCE_STDLIBS')
 
 
+class libbulkmemory(MuslInternalLibrary,
+                    DebugLibrary,
+                    AsanInstrumentedLibrary,
+                    MTLibrary):
+  name = 'libbulkmemory'
+  src_dir = 'system/lib/libc'
+  src_files = ['emscripten_memcpy.c', 'emscripten_memset.c',
+               'emscripten_memcpy_big.S', 'emscripten_memset_big.S']
+  cflags = ['-mbulk-memory']
+
+  def can_use(self):
+    return super(libbulkmemory, self).can_use() and settings.BULK_MEMORY
+
+
 class libprintf_long_double(libc):
   name = 'libprintf_long_double'
   cflags = ['-DEMSCRIPTEN_PRINTF_LONG_DOUBLE']
@@ -1945,7 +1959,7 @@ def get_files(self):
                    '__main_void.c'])
     files += files_in_path(
         path='system/lib/libc',
-        filenames=['emscripten_memcpy.c'])
+        filenames=['emscripten_memcpy.c', 'emscripten_memset.c'])
     # It is more efficient to use JS methods for time, normally.
     files += files_in_path(
         path='system/lib/libc/musl/src/time',
@@ -2154,7 +2168,8 @@ def add_sanitizer_libs():
   if settings.SHRINK_LEVEL >= 2 and not settings.LINKABLE and \
      not os.environ.get('EMCC_FORCE_STDLIBS'):
     add_library('libc_optz')
-
+  if settings.BULK_MEMORY:
+    add_library('libbulkmemory')
   if settings.STANDALONE_WASM:
     add_library('libstandalonewasm')
   if settings.ALLOW_UNIMPLEMENTED_SYSCALLS: