file_packager.py: Add option to embed file data in wasm binary (#16050)

This change not only adds the new option but uses this option
whenever file_packager is used from within emcc.

Hopefully we can find a way deprecate and remove the old JS
embedded since that seems strictly worse in almost ever way.

- Larger code size (JS base64 encoding is larger than binary)
- No possibility of zero copy, memory-backed files
- Less compatible with standalone wasm / WASI

Author: sbc100

ChangeLog.md

@@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works.
 
 3.1.3
 -----
+- The file packager now supports embedding files directly into wasm memory and
+  `emcc` now uses this mode when the `--embed-file` option is used.  If you
+  use `file_packager` directly it is recommended that you switch to the new mode
+  by adding `--obj-output` to the command line. (#16050)
 
 3.1.2 - 20/01/2022
 ------------------

emcc.py

@@ -814,14 +814,7 @@ def array_contains_any_of(hay, needles):
 
 
 def get_clang_flags():
-  return ['-target', get_llvm_target()]
-
-
-def get_llvm_target():
-  if settings.MEMORY64:
-    return 'wasm64-unknown-emscripten'
-  else:
-    return 'wasm32-unknown-emscripten'
+  return ['-target', shared.get_llvm_target()]
 
 
 cflags = None
@@ -979,6 +972,41 @@ def get_subresource_location(path, data_uri=None):
     return os.path.basename(path)
 
 
+@ToolchainProfiler.profile_block('package_files')
+def package_files(options, target):
+  rtn = []
+  logger.debug('setting up files')
+  file_args = ['--from-emcc', '--export-name=' + settings.EXPORT_NAME]
+  if options.preload_files:
+    file_args.append('--preload')
+    file_args += options.preload_files
+  if options.embed_files:
+    file_args.append('--embed')
+    file_args += options.embed_files
+  if options.exclude_files:
+    file_args.append('--exclude')
+    file_args += options.exclude_files
+  if options.use_preload_cache:
+    file_args.append('--use-preload-cache')
+  if settings.LZ4:
+    file_args.append('--lz4')
+  if options.use_preload_plugins:
+    file_args.append('--use-preload-plugins')
+  if not settings.ENVIRONMENT_MAY_BE_NODE:
+    file_args.append('--no-node')
+  if options.embed_files:
+    object_file = in_temp('embedded_files.o')
+    file_args += ['--obj-output=' + object_file]
+    rtn.append(object_file)
+
+  cmd = [shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args
+  file_code = shared.check_call(cmd, stdout=PIPE).stdout
+
+  options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code)
+
+  return rtn
+
+
 run_via_emxx = False
 
 
@@ -1053,7 +1081,7 @@ def run(args):
     return 0
 
   if '-dumpmachine' in args:
-    print(get_llvm_target())
+    print(shared.get_llvm_target())
     return 0
 
   if '-dumpversion' in args: # gcc's doc states "Print the compiler version [...] and don't do anything else."
@@ -1129,6 +1157,10 @@ def run(args):
   # Link object files using wasm-ld or llvm-link (for bitcode linking)
   linker_arguments = phase_calculate_linker_inputs(options, state, linker_inputs)
 
+  # Embed and preload files
+  if len(options.preload_files) or len(options.embed_files):
+    linker_arguments += package_files(options, target)
+
   if options.oformat == OFormat.OBJECT:
     logger.debug(f'link_to_object: {linker_arguments} -> {target}')
     building.link_to_object(linker_arguments, target)
@@ -2700,7 +2732,7 @@ def phase_post_link(options, state, in_wasm, wasm_target, target):
 
   phase_emscript(options, in_wasm, wasm_target, memfile)
 
-  phase_source_transforms(options, target)
+  phase_source_transforms(options)
 
   if memfile and not settings.MINIMAL_RUNTIME:
     # MINIMAL_RUNTIME doesn't use `var memoryInitializer` but instead expects Module['mem'] to
@@ -2731,33 +2763,9 @@ def phase_emscript(options, in_wasm, wasm_target, memfile):
 
 
 @ToolchainProfiler.profile_block('source transforms')
-def phase_source_transforms(options, target):
+def phase_source_transforms(options):
   global final_js
 
-  # Embed and preload files
-  if len(options.preload_files) or len(options.embed_files):
-    logger.debug('setting up files')
-    file_args = ['--from-emcc', '--export-name=' + settings.EXPORT_NAME]
-    if len(options.preload_files):
-      file_args.append('--preload')
-      file_args += options.preload_files
-    if len(options.embed_files):
-      file_args.append('--embed')
-      file_args += options.embed_files
-    if len(options.exclude_files):
-      file_args.append('--exclude')
-      file_args += options.exclude_files
-    if options.use_preload_cache:
-      file_args.append('--use-preload-cache')
-    if settings.LZ4:
-      file_args.append('--lz4')
-    if options.use_preload_plugins:
-      file_args.append('--use-preload-plugins')
-    if not settings.ENVIRONMENT_MAY_BE_NODE:
-      file_args.append('--no-node')
-    file_code = shared.check_call([shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args, stdout=PIPE).stdout
-    options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code)
-
   # Apply pre and postjs files
   if final_js and (options.pre_js or options.post_js):
     logger.debug('applying pre/postjses')

src/library_fs.js

@@ -1590,7 +1590,11 @@ FS.staticInit();` +
       return FS.create(path, mode);
     },
     createDataFile: function(parent, name, data, canRead, canWrite, canOwn) {
-      var path = name ? PATH.join2(typeof parent === 'string' ? parent : FS.getPath(parent), name) : parent;
+      var path = name;
+      if (parent) {
+        parent = typeof parent === 'string' ? parent : FS.getPath(parent);
+        path = name ? PATH.join2(parent, name) : parent;
+      }
       var mode = FS.getMode(canRead, canWrite);
       var node = FS.create(path, mode);
       if (data) {

tests/other/metadce/hello_world_O3_MAIN_MODULE_2.funcs

@@ -3,8 +3,9 @@ $__emscripten_stdout_seek
 $__fwritex
 $__stdio_write
 $__towrite
-$__wasm_apply_global_relocs
+$__wasm_apply_data_relocs
 $__wasm_call_ctors
+$__wasm_start
 $dlmalloc
 $legalstub$dynCall_jiji
 $main

tests/test_other.py

@@ -2590,7 +2590,7 @@ def test_file_packager_directory_with_single_quote(self):
     assert json.dumps("direc'tory") in proc.stdout
 
   def test_file_packager_mention_FORCE_FILESYSTEM(self):
-    MESSAGE = 'Remember to build the main file with  -s FORCE_FILESYSTEM=1  so that it includes support for loading this file package'
+    MESSAGE = 'Remember to build the main file with `-sFORCE_FILESYSTEM` so that it includes support for loading this file package'
     create_file('data.txt', 'data1')
     # mention when running standalone
     err = self.run_process([FILE_PACKAGER, 'test.data', '--preload', 'data.txt'], stdout=PIPE, stderr=PIPE).stderr
@@ -2608,7 +2608,11 @@ def test_file_packager_returns_error_if_target_equal_to_jsoutput(self):
   def test_file_packager_embed(self):
     create_file('data.txt', 'hello data')
 
-    self.run_process([FILE_PACKAGER, 'test.data', '--embed', 'data.txt', '--js-output=data.js'])
+    # Without --obj-output we issue a warning
+    err = self.run_process([FILE_PACKAGER, 'test.data', '--embed', 'data.txt', '--js-output=data.js'], stderr=PIPE).stderr
+    self.assertContained('--obj-output is recommended when using --embed', err)
+
+    self.run_process([FILE_PACKAGER, 'test.data', '--embed', 'data.txt', '--obj-output=data.o', '--js-output=data.js'])
 
     create_file('test.c', '''
     #include <stdio.h>
@@ -2623,7 +2627,7 @@ def test_file_packager_embed(self):
       return 0;
     }
     ''')
-    self.run_process([EMCC, '--pre-js=data.js', 'test.c', '-sFORCE_FILESYSTEM'])
+    self.run_process([EMCC, '--pre-js=data.js', 'test.c', 'data.o', '-sFORCE_FILESYSTEM'])
     output = self.run_js('a.out.js')
     self.assertContained('hello data', output)