Add suppport for external ports (#21316)

Author: ypujante

ChangeLog.md

@@ -42,6 +42,8 @@ See docs/process.md for more on how version tagging works.
   available via `--use-port=contrib.glfw3`: an emscripten port of glfw written 
   in C++ with many features like support for multiple windows. (#21244 and 
   #21276)
+- Added concept of external ports which live outside emscripten and are
+  loaded on demand using the syntax `--use-port=/path/to/my_port.py` (#21316)
 
 
 3.1.53 - 01/29/24

site/source/docs/compiling/Building-Projects.rst

@@ -278,6 +278,15 @@ The simplest way to add a new port is to put it under the ``contrib`` directory.
  * Make sure the port is open source and has a suitable license.
  * Read the ``README.md`` file under ``tools/ports/contrib`` which contains more information.
 
+External ports
+--------------
+
+Emscripten also supports external ports (ports that are not part of the
+distribution). In order to use such a port, you simply provide its path:
+``--use-port=/path/to/my_port.py``
+
+.. note:: Be aware that if you are working on the code of a port, the port API
+  used by emscripten is not 100% stable and could change between versions.
 
 Build system issues
 ===================

test/other/ports/external.py

@@ -0,0 +1,62 @@
+# Copyright 2024 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+
+import os
+from typing import Dict, Optional
+
+OPTIONS = {
+  'value1': 'Value for define TEST_VALUE_1',
+  'value2': 'Value for define TEST_VALUE_2',
+  'dependency': 'A dependency'
+}
+
+# user options (from --use-port)
+opts: Dict[str, Optional[str]] = {
+  'value1': None,
+  'value2': None,
+  'dependency': None
+}
+
+deps = []
+
+
+def get_lib_name(settings):
+  return 'lib_external.a'
+
+
+def get(ports, settings, shared):
+  # for simplicity in testing, the source is in the same folder as the port and not fetched as a tarball
+  source_dir = os.path.dirname(__file__)
+
+  def create(final):
+    ports.install_headers(source_dir)
+    print(f'about to build {source_dir}')
+    ports.build_port(source_dir, final, 'external')
+
+  return [shared.cache.get_lib(get_lib_name(settings), create, what='port')]
+
+
+def clear(ports, settings, shared):
+  shared.cache.erase_lib(get_lib_name(settings))
+
+
+def process_args(ports):
+  args = ['-isystem', ports.get_include_dir('external')]
+  if opts['value1']:
+    args.append(f'-DTEST_VALUE_1={opts["value1"]}')
+  if opts['value2']:
+    args.append(f'-DTEST_VALUE_2={opts["value2"]}')
+  if opts['dependency']:
+    args.append(f'-DTEST_DEPENDENCY_{opts["dependency"].upper()}')
+  return args
+
+
+def process_dependencies(settings):
+  if opts['dependency']:
+    deps.append(opts['dependency'])
+
+
+def handle_options(options):
+  opts.update(options)

test/other/ports/my_port.c

@@ -0,0 +1,3 @@
+int my_port_fn(int value) {
+  return value;
+}

test/other/ports/my_port.h

@@ -0,0 +1 @@
+int my_port_fn(int);

test/other/ports/simple.py

@@ -0,0 +1,25 @@
+# Copyright 2024 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+
+import os
+
+
+def get_lib_name(settings):
+  return 'lib_simple.a'
+
+
+def get(ports, settings, shared):
+  # for simplicity in testing, the source is in the same folder as the port and not fetched as a tarball
+  source_dir = os.path.dirname(__file__)
+
+  def create(final):
+    ports.install_headers(source_dir)
+    ports.build_port(source_dir, final, 'simple')
+
+  return [shared.cache.get_lib(get_lib_name(settings), create, what='port')]
+
+
+def clear(ports, settings, shared):
+  shared.cache.erase_lib(get_lib_name(settings))

test/other/test_external_ports.c

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2024 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#include <my_port.h>
+#include <assert.h>
+#include <stdio.h>
+
+#ifdef TEST_DEPENDENCY_SDL2
+#include <SDL2/SDL.h>
+#endif
+
+// TEST_VALUE_1 and TEST_VALUE_2 are defined via port options
+#ifndef TEST_VALUE_1
+#define TEST_VALUE_1 0
+#endif
+#ifndef TEST_VALUE_2
+#define TEST_VALUE_2 0
+#endif
+
+int main() {
+  assert(my_port_fn(99) == 99); // check that we can call a function from my_port.h
+  printf("value1=%d&value2=%d\n", TEST_VALUE_1, TEST_VALUE_2);
+#ifdef TEST_DEPENDENCY_SDL2
+  SDL_version version;
+  SDL_VERSION(&version);
+  printf("sdl2=%d\n", version.major);
+#endif
+  return 0;
+}

test/other/test_external_ports_simple.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2024 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#include <my_port.h>
+#include <assert.h>
+
+int main() {
+  assert(my_port_fn(99) == 99); // check that we can call a function from my_port.h
+  return 0;
+}

test/test_other.py

@@ -2393,6 +2393,31 @@ def test_contrib_ports(self):
     # with a different contrib port when there is another one
     self.emcc(test_file('other/test_contrib_ports.cpp'), ['--use-port=contrib.glfw3'])
 
+  @crossplatform
+  def test_external_ports_simple(self):
+    if config.FROZEN_CACHE:
+      self.skipTest("test doesn't work with frozen cache")
+    simple_port_path = test_file("other/ports/simple.py")
+    self.do_runf('other/test_external_ports_simple.c', emcc_args=[f'--use-port={simple_port_path}'])
+
+  @crossplatform
+  def test_external_ports(self):
+    if config.FROZEN_CACHE:
+      self.skipTest("test doesn't work with frozen cache")
+    external_port_path = test_file("other/ports/external.py")
+    # testing no option
+    self.do_runf('other/test_external_ports.c', 'value1=0&value2=0\n', emcc_args=[f'--use-port={external_port_path}'])
+    # testing 1 option
+    self.do_runf('other/test_external_ports.c', 'value1=12&value2=0\n', emcc_args=[f'--use-port={external_port_path}:value1=12'])
+    # testing 2 options
+    self.do_runf('other/test_external_ports.c', 'value1=12&value2=36\n', emcc_args=[f'--use-port={external_port_path}:value1=12:value2=36'])
+    # testing dependency
+    self.do_runf('other/test_external_ports.c', 'sdl2=2\n', emcc_args=[f'--use-port={external_port_path}:dependency=sdl2'])
+    # testing invalid dependency
+    stderr = self.expect_fail([EMCC, test_file('other/test_external_ports.c'), f'--use-port={external_port_path}:dependency=invalid', '-o', 'a4.out.js'])
+    self.assertFalse(os.path.exists('a4.out.js'))
+    self.assertContained('Unknown dependency `invalid` for port `external`', stderr)
+
   def test_link_memcpy(self):
     # memcpy can show up *after* optimizations, so after our opportunity to link in libc, so it must be special-cased
     create_file('main.c', r'''
@@ -14528,16 +14553,16 @@ def test_js_preprocess_pre_post(self):
   def test_use_port_errors(self, compiler):
     stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=invalid', '-o', 'out.js'])
     self.assertFalse(os.path.exists('out.js'))
-    self.assertContained('Error with --use-port=invalid | invalid port name: invalid', stderr)
+    self.assertContained('Error with `--use-port=invalid` | invalid port name: `invalid`', stderr)
     stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=sdl2:opt1=v1', '-o', 'out.js'])
     self.assertFalse(os.path.exists('out.js'))
-    self.assertContained('Error with --use-port=sdl2:opt1=v1 | no options available for port sdl2', stderr)
+    self.assertContained('Error with `--use-port=sdl2:opt1=v1` | no options available for port `sdl2`', stderr)
     stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=sdl2_image:format=jpg', '-o', 'out.js'])
     self.assertFalse(os.path.exists('out.js'))
-    self.assertContained('Error with --use-port=sdl2_image:format=jpg | format is not supported', stderr)
+    self.assertContained('Error with `--use-port=sdl2_image:format=jpg` | `format` is not supported', stderr)
     stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=sdl2_image:formats', '-o', 'out.js'])
     self.assertFalse(os.path.exists('out.js'))
-    self.assertContained('Error with --use-port=sdl2_image:formats | formats is missing a value', stderr)
+    self.assertContained('Error with `--use-port=sdl2_image:formats` | `formats` is missing a value', stderr)
     stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=sdl2_image:formats=jpg:formats=png', '-o', 'out.js'])
     self.assertFalse(os.path.exists('out.js'))
-    self.assertContained('Error with --use-port=sdl2_image:formats=jpg:formats=png | duplicate option formats', stderr)
+    self.assertContained('Error with `--use-port=sdl2_image:formats=jpg:formats=png` | duplicate option `formats`', stderr)

tools/ports/__init__.py

@@ -8,6 +8,8 @@
 import os
 import shutil
 import glob
+import importlib.util
+import sys
 from typing import Set
 from tools import cache
 from tools import config
@@ -33,8 +35,7 @@
 logger = logging.getLogger('ports')
 
 
-def load_port(name):
-  port = __import__(name, globals(), level=1, fromlist=[None])
+def init_port(name, port):
   ports.append(port)
   port.is_contrib = name.startswith('contrib.')
   port.name = name
@@ -60,9 +61,29 @@ def load_port(name):
 
   for variant, extra_settings in port.variants.items():
     if variant in port_variants:
-      utils.exit_with_error('duplicate port variant: %s' % variant)
+      utils.exit_with_error('duplicate port variant: `%s`' % variant)
     port_variants[variant] = (port.name, extra_settings)
 
+  validate_port(port)
+
+
+def load_port_by_name(name):
+  port = __import__(name, globals(), level=1, fromlist=[None])
+  init_port(name, port)
+
+
+def load_port_by_path(path):
+  name = os.path.splitext(os.path.basename(path))[0]
+  if name in ports_by_name:
+    utils.exit_with_error(f'port path [`{path}`] is invalid: duplicate port name `{name}`')
+  module_name = f'tools.ports.{name}'
+  spec = importlib.util.spec_from_file_location(module_name, path)
+  port = importlib.util.module_from_spec(spec)
+  sys.modules[module_name] = port
+  spec.loader.exec_module(port)
+  init_port(name, port)
+  return name
+
 
 def validate_port(port):
   expected_attrs = ['get', 'clear', 'show']
@@ -74,30 +95,20 @@ def validate_port(port):
     assert hasattr(port, a), 'port %s is missing %s' % (port, a)
 
 
-def validate_ports():
-  for port in ports:
-    validate_port(port)
-    for dep in port.deps:
-      if dep not in ports_by_name:
-        utils.exit_with_error('unknown dependency in port: %s' % dep)
-
-
 @ToolchainProfiler.profile()
 def read_ports():
   for filename in os.listdir(ports_dir):
     if not filename.endswith('.py') or filename == '__init__.py':
       continue
     filename = os.path.splitext(filename)[0]
-    load_port(filename)
+    load_port_by_name(filename)
 
   contrib_dir = os.path.join(ports_dir, 'contrib')
   for filename in os.listdir(contrib_dir):
     if not filename.endswith('.py') or filename == '__init__.py':
       continue
     filename = os.path.splitext(filename)[0]
-    load_port('contrib.' + filename)
-
-  validate_ports()
+    load_port_by_name('contrib.' + filename)
 
 
 def get_all_files_under(dirname):
@@ -386,6 +397,8 @@ def resolve_dependencies(port_set, settings):
   def add_deps(node):
     node.process_dependencies(settings)
     for d in node.deps:
+      if d not in ports_by_name:
+        utils.exit_with_error(f'Unknown dependency `{d}` for port `{node.name}`')
       dep = ports_by_name[d]
       if dep not in port_set:
         port_set.add(dep)
@@ -396,31 +409,38 @@ def add_deps(node):
 
 
 def handle_use_port_error(arg, message):
-  utils.exit_with_error(f'Error with --use-port={arg} | {message}')
+  utils.exit_with_error(f'Error with `--use-port={arg}` | {message}')
 
 
 def handle_use_port_arg(settings, arg):
-  args = arg.split(':', 1)
-  name, options = args[0], None
-  if len(args) == 2:
-    options = args[1]
-  if name not in ports_by_name:
-    handle_use_port_error(arg, f'invalid port name: {name}')
+  # Ignore ':' in first or second char of string since we could be dealing with a windows drive separator
+  pos = arg.find(':', 2)
+  if pos != -1:
+    name, options = arg[:pos], arg[pos + 1:]
+  else:
+    name, options = arg, None
+  if name.endswith('.py'):
+    port_file_path = name
+    if not os.path.isfile(port_file_path):
+      handle_use_port_error(arg, f'not a valid port path: {port_file_path}')
+    name = load_port_by_path(port_file_path)
+  elif name not in ports_by_name:
+    handle_use_port_error(arg, f'invalid port name: `{name}`')
   ports_needed.add(name)
   if options:
     port = ports_by_name[name]
     if not hasattr(port, 'handle_options'):
-      handle_use_port_error(arg, f'no options available for port {name}')
+      handle_use_port_error(arg, f'no options available for port `{name}`')
     else:
       options_dict = {}
       for name_value in options.split(':'):
         nv = name_value.split('=', 1)
         if len(nv) != 2:
-          handle_use_port_error(arg, f'{name_value} is missing a value')
+          handle_use_port_error(arg, f'`{name_value}` is missing a value')
         if nv[0] not in port.OPTIONS:
-          handle_use_port_error(arg, f'{nv[0]} is not supported; available options are {port.OPTIONS}')
+          handle_use_port_error(arg, f'`{nv[0]}` is not supported; available options are {port.OPTIONS}')
         if nv[0] in options_dict:
-          handle_use_port_error(arg, f'duplicate option {nv[0]}')
+          handle_use_port_error(arg, f'duplicate option `{nv[0]}`')
         options_dict[nv[0]] = nv[1]
       port.handle_options(options_dict)