Merge remote-tracking branch 'origin/main' into roll

Author: kripken

.circleci/config.yml

@@ -753,6 +753,7 @@ jobs:
           test_targets: "
             other.test_gen_struct_info
             other.test_native_call_before_init
+            other.test_js_optimizer_verbose
             other.test_node_unhandled_rejection
             other.test_full_js_library*
             core2.test_hello_world"
@@ -768,6 +769,7 @@ jobs:
             other.test_gen_struct_info
             other.test_native_call_before_init
             other.test_node_unhandled_rejection
+            other.test_js_optimizer_verbose
             other.test_min_node_version
             other.test_node_emscripten_num_logical_cores
             core2.test_pthread_create

src/library_getvalue.js

@@ -62,7 +62,7 @@ var LibraryMemOps = {
 
 #if SAFE_HEAP
   // The same as the above two functions, but known to the safeHeap pass
-  // in tools/acorn-optimizer.js.  The heap accesses within these two
+  // in tools/acorn-optimizer.mjs.  The heap accesses within these two
   // functions will *not* get re-written.
   // Note that we do not use the alias mechanism here since we need separate
   // instances of above setValueImpl/getValueImpl functions.

test/common.py

@@ -28,6 +28,7 @@
 import time
 import webbrowser
 import unittest
+import queue
 
 import clang_native
 import jsrun
@@ -76,6 +77,7 @@
 NON_ZERO = -1
 
 TEST_ROOT = path_from_root('test')
+LAST_TEST = path_from_root('out/last_test.txt')
 
 WEBIDL_BINDER = shared.bat_suffix(path_from_root('tools/webidl_binder'))
 
@@ -901,6 +903,7 @@ def setUp(self):
       else:
         print('Creating new test output directory')
         ensure_dir(self.working_dir)
+      utils.write_file(LAST_TEST, self.id() + '\n')
     os.chdir(self.working_dir)
 
   def runningInParallel(self):
@@ -1956,25 +1959,21 @@ def run_browser(self, html_file, expected=None, message=None, timeout=None, extr
           'http://localhost:%s/%s' % (self.port, html_file),
           self.get_dir()
         ))
-        received_output = False
-        output = '[no http server activity]'
-        start = time.time()
         if timeout is None:
           timeout = self.browser_timeout
-        while time.time() - start < timeout:
-          if not self.harness_out_queue.empty():
-            output = self.harness_out_queue.get()
-            received_output = True
-            break
-          time.sleep(0.1)
-        if not received_output:
+        try:
+          output = self.harness_out_queue.get(block=True, timeout=timeout)
+        except queue.Empty:
           BrowserCore.unresponsive_tests += 1
           print('[unresponsive tests: %d]' % BrowserCore.unresponsive_tests)
           self.browser_restart()
+          # Rather than fail the test here, let fail on the `assertContained` so
+          # that the test can be retried via `extra_tries`
+          output = '[no http server activity]'
         if output is None:
           # the browser harness reported an error already, and sent a None to tell
           # us to also fail the test
-          raise Exception('failing test due to browser harness error')
+          self.fail('browser harness error')
         if output.startswith('/report_result?skipped:'):
           self.skipTest(unquote(output[len('/report_result?skipped:'):]).strip())
         else:

test/runner.py

@@ -382,6 +382,9 @@ def parse_args(args):
   parser.add_argument('tests', nargs='*')
   parser.add_argument('--failfast', action='store_const', const=True, default=False)
   parser.add_argument('--start-at', metavar='NAME', help='Skip all tests up until <NAME>')
+  parser.add_argument('--continue', dest='_continue', action='store_true',
+                      help='Resume from the last run test.'
+                           'Useful when combined with --failfast')
   parser.add_argument('--force64', action='store_const', const=True, default=None)
   parser.add_argument('--crossplatform-only', action='store_true')
   return parser.parse_args()
@@ -465,6 +468,11 @@ def prepend_default(arg):
     tests = tests_with_expanded_wildcards(tests, all_tests)
     tests = skip_requested_tests(tests, modules)
     tests = args_for_random_tests(tests, modules)
+
+  if not options.start_at and options._continue:
+    if os.path.exists(common.LAST_TEST):
+      options.start_at = utils.read_file(common.LAST_TEST).strip()
+
   suites, unmatched_tests = load_test_suites(tests, modules, options.start_at)
   if unmatched_tests:
     print('ERROR: could not find the following tests: ' + ' '.join(unmatched_tests))

test/test_core.py

@@ -597,71 +597,7 @@ def test_cube2hash(self):
                          ('64bitisslow', '64D8470573635EC354FEE7B7F87C566FCAF1EFB491041670')]:
       self.do_run('src.js', 'hash value: ' + output, args=[text], no_build=True)
 
-  def test_unaligned(self):
-    self.skipTest('LLVM marks the reads of s as fully aligned, making this test invalid')
-    src = r'''
-      #include <stdio.h>
-
-      struct S {
-        double x;
-        int y;
-      };
-
-      int main() {
-        // the 64-bit value here will not be 8-byte aligned
-        S s0[3] = { {0x12a751f430142, 22}, {0x17a5c85bad144, 98}, {1, 1}};
-        char buffer[10*sizeof(S)];
-        int b = int(buffer);
-        S *s = (S*)(b + 4-b%8);
-        s[0] = s0[0];
-        s[1] = s0[1];
-        s[2] = s0[2];
-
-        printf("*%d : %d : %d\n", sizeof(S), ((unsigned long)&s[0]) % 8 != ((unsigned long)&s[1]) % 8,
-                                             ((unsigned long)&s[1]) - ((unsigned long)&s[0]));
-        s[0].x++;
-        s[0].y++;
-        s[1].x++;
-        s[1].y++;
-        printf("%.1f,%d,%.1f,%d\n", s[0].x, s[0].y, s[1].x, s[1].y);
-        return 0;
-      }
-      '''
-
-    # TODO: A version of this with int64s as well
-
-    self.do_run(src, '*12 : 1 : 12\n328157500735811.0,23,416012775903557.0,99\n')
-
-    return # TODO: continue to the next part here
-
-    # Test for undefined behavior in C. This is not legitimate code, but does exist
-
-    src = r'''
-      #include <stdio.h>
-
-      int main()
-      {
-        int x[10];
-        char *p = (char*)&x[0];
-        p++;
-        short *q = (short*)p;
-        *q = 300;
-        printf("*%d:%ld*\n", *q, ((long)q)%2);
-        int *r = (int*)p;
-        *r = 515559;
-        printf("*%d*\n", *r);
-        long long *t = (long long*)p;
-        *t = 42949672960;
-        printf("*%lld*\n", *t);
-        return 0;
-      }
-    '''
-
-    try:
-      self.do_run(src, '*300:1*\n*515559*\n*42949672960*\n')
-    except Exception as e:
-      assert 'must be aligned' in str(e), e # expected to fail without emulation
-
+  @only_wasm2js('tests 64-bit alignment of structs')
   def test_align64(self):
     src = r'''
       #include <stdio.h>
@@ -710,12 +646,7 @@ def test_align64(self):
 0.00,10,123.46,0.00 : 0.00,10,123.46,0.00
 ''')
 
-  @no_asan('asan errors on corner cases we check')
-  @no_lsan('lsan errors on corner cases we check')
-  def test_aligned_alloc(self):
-    self.do_runf('test_aligned_alloc.c', '',
-                 emcc_args=['-Wno-non-power-of-two-alignment'])
-
+  @only_wasm2js('tests signed vs unsigned values')
   def test_unsigned(self):
     src = '''
       #include <stdio.h>
@@ -801,22 +732,26 @@ def test_unsigned(self):
     '''
     self.do_run(src, '*255*\n*65535*\n*-1*\n*-1*\n*-1*')
 
+  @only_wasm2js('tests 1-bit fields')
   def test_bitfields(self):
     self.do_core_test('test_bitfields.c')
 
   def test_floatvars(self):
     self.do_core_test('test_floatvars.cpp')
 
+  @only_wasm2js('tests pointer casts')
   def test_closebitcasts(self):
     self.do_core_test('closebitcasts.c')
 
   def test_fast_math(self):
     self.emcc_args += ['-ffast-math']
     self.do_core_test('test_fast_math.c', args=['5', '6', '8'])
 
+  @only_wasm2js('tests division by zero')
   def test_zerodiv(self):
     self.do_core_test('test_zerodiv.c')
 
+  @only_wasm2js('tests multiplication by zero')
   def test_zero_multiplication(self):
     # needs to flush stdio streams
     self.set_setting('EXIT_RUNTIME')

test/test_other.py

@@ -2617,7 +2617,7 @@ def test_js_optimizer(self, input, passes):
     input = test_file(input)
     expected_file = os.path.splitext(input)[0] + '-output.js'
     # test calling optimizer
-    js = self.run_process(config.NODE_JS + [path_from_root('tools/acorn-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).stdout
+    js = self.run_process(config.NODE_JS + [path_from_root('tools/acorn-optimizer.mjs'), input] + passes, stdin=PIPE, stdout=PIPE).stdout
     if common.EMTEST_REBASELINE:
       write_file(expected_file, js)
     else:
@@ -14241,3 +14241,7 @@ def test_sysroot_includes_first(self):
   def test_force_filesystem_error(self):
     err = self.expect_fail([EMCC, test_file('hello_world.c'), '-sFILESYSTEM=0', '-sFORCE_FILESYSTEM'])
     self.assertContained('emcc: error: `-sFORCE_FILESYSTEM` cannot be used with `-sFILESYSTEM=0`', err)
+
+  def test_aligned_alloc(self):
+    self.do_runf('test_aligned_alloc.c', '',
+                 emcc_args=['-Wno-non-power-of-two-alignment'])

third_party/terser/terser.js

@@ -6706,7 +6706,7 @@ const _NOINLINE   = 0b00000100;
 const _KEY        = 0b00001000;
 const _MANGLEPROP = 0b00010000;
 
-// XXX Emscripten: export TreeWalker for walking through AST in acorn-optimizer.js.
+// XXX Emscripten: export TreeWalker for walking through AST in acorn-optimizer.mjs.
 exports.TreeWalker = TreeWalker;
 
 /***********************************************************************

tools/acorn-optimizer.js renamed to tools/acorn-optimizer.mjs

@@ -1,10 +1,8 @@
 #!/usr/bin/env node
 
-'use strict';
-
-const acorn = require('acorn');
-const terser = require('../third_party/terser/terser');
-const fs = require('fs');
+import * as acorn from 'acorn';
+import * as terser from '../third_party/terser/terser.js';
+import * as fs from 'fs';
 
 // Utilities
 

tools/building.py

@@ -349,7 +349,7 @@ def js_optimizer(filename, passes):
 
 # run JS optimizer on some JS, ignoring asm.js contents if any - just run on it all
 def acorn_optimizer(filename, passes, extra_info=None, return_output=False):
-  optimizer = path_from_root('tools/acorn-optimizer.js')
+  optimizer = path_from_root('tools/acorn-optimizer.mjs')
   original_filename = filename
   if extra_info is not None:
     temp_files = shared.get_temp_files()
@@ -579,7 +579,7 @@ def closure_compiler(filename, advanced=True, extra_closure_args=None):
     CLOSURE_EXTERNS += [path_from_root('src/closure-externs/minimal_runtime_worker_externs.js')]
 
   args = ['--compilation_level', 'ADVANCED_OPTIMIZATIONS' if advanced else 'SIMPLE_OPTIMIZATIONS']
-  # Keep in sync with ecmaVersion in tools/acorn-optimizer.js
+  # Keep in sync with ecmaVersion in tools/acorn-optimizer.mjs
   args += ['--language_in', 'ECMASCRIPT_2021']
   # Tell closure not to do any transpiling or inject any polyfills.
   # At some point we may want to look into using this as way to convert to ES5 but

tools/js_optimizer.py

@@ -22,7 +22,7 @@
 temp_files = shared.get_temp_files()
 
 
-ACORN_OPTIMIZER = path_from_root('tools/acorn-optimizer.js')
+ACORN_OPTIMIZER = path_from_root('tools/acorn-optimizer.mjs')
 
 NUM_CHUNKS_PER_CORE = 3
 MIN_CHUNK_SIZE = int(os.environ.get('EMCC_JSOPT_MIN_CHUNK_SIZE') or 512 * 1024) # configuring this is just for debugging purposes
@@ -65,7 +65,7 @@ def split_funcs(js):
 
 class Minifier:
   """minification support. We calculate minification of
-  globals here, then pass that into the parallel acorn-optimizer.js runners which
+  globals here, then pass that into the parallel acorn-optimizer.mjs runners which
   perform minification of locals.
   """
 
@@ -75,7 +75,7 @@ def __init__(self, js):
     self.profiling_funcs = False
 
   def minify_shell(self, shell, minify_whitespace):
-    # Run through acorn-optimizer.js to find and minify the global symbols
+    # Run through acorn-optimizer.mjs to find and minify the global symbols
     # We send it the globals, which it parses at the proper time. JS decides how
     # to minify all global names, we receive a dictionary back, which is then
     # used by the function processors

tools/preprocessor.js renamed to tools/preprocessor.mjs

@@ -14,24 +14,29 @@
 
 'use strict';
 
-const fs = require('fs');
-const path = require('path');
-global.vm = require('vm');
+import * as fs from 'fs';
+import * as path from 'path';
+import * as vm from 'vm';
+import assert from 'assert';
+import * as url from 'url';
 
-const arguments_ = process.argv.slice(2);
+const args = process.argv.slice(2);
 const debug = false;
 
+// Anything needed by the script that we load below must be added to the
+// global object.  These, for example, are all needed by parseTools.js.
+global.vm = vm;
+global.assert = assert;
 global.print = (x) => {
   process.stdout.write(x + '\n');
 };
 global.printErr = (x) => {
   process.stderr.write(x + '\n');
 };
 
-global.assert = require('assert');
-
 function find(filename) {
-  const prefixes = [process.cwd(), path.join(__dirname, '..', 'src')];
+  const dirname = url.fileURLToPath(new URL('.', import.meta.url));
+  const prefixes = [process.cwd(), path.join(dirname, '..', 'src')];
   for (let i = 0; i < prefixes.length; ++i) {
     const combined = path.join(prefixes[i], filename);
     if (fs.existsSync(combined)) {
@@ -50,9 +55,10 @@ global.load = (f) => {
   (0, eval)(read(f) + '//# sourceURL=' + find(f));
 };
 
-const settingsFile = arguments_[0];
-const inputFile = arguments_[1];
-const expandMacros = arguments_.includes('--expandMacros');
+assert(args.length >= 2);
+const settingsFile = args[0];
+const inputFile = args[1];
+const expandMacros = args.includes('--expandMacros');
 
 load(settingsFile);
 load('utility.js');

tools/shared.py

@@ -762,7 +762,7 @@ def read_and_preprocess(filename, expand_macros=False):
   if expand_macros:
     args += ['--expandMacros']
 
-  run_js_tool(path_from_root('tools/preprocessor.js'), args, stdout=open(stdout, 'w'), cwd=dirname)
+  run_js_tool(path_from_root('tools/preprocessor.mjs'), args, stdout=open(stdout, 'w'), cwd=dirname)
   out = utils.read_file(stdout)
 
   return out