(chore) Builds ES and CJS modules side by side with conditional exports (#3007)

- drops the legacy `lib/highlight.js`
- build ES modules in `es` folder side-by-side with `lib` for CJS
- adds `lib/common`
- loader is no longer supported, languages must `export default` or equiv
- add --no-esm build option (for speed)
- add `.js.js` stubs for maximum compatible with requires using extensions

Author: joshgoebel

README.md

@@ -255,11 +255,16 @@ const hljs = require('./highlight.js');
 const highlightedCode = hljs.highlightAuto('<span>Hello World!</span>').value
 ```
 
-For a smaller footprint, load only the languages you need.
+For a smaller footprint, load our common subset of languages (the same set used for our default web build).
 
 ```js
-const hljs = require('highlight.js/lib/core');  // require only the core library
-// separately require languages
+const hljs = require('highlight.js/lib/common');
+```
+
+For the smallest footprint, load only the languages you need:
+
+```js
+const hljs = require('highlight.js/lib/core');
 hljs.registerLanguage('xml', require('highlight.js/lib/languages/xml'));
 
 const highlightedCode = hljs.highlight('<span>Hello World!</span>', {language: 'xml'}).value

package.json

@@ -32,6 +32,7 @@
     "type": "git",
     "url": "git://github.com/highlightjs/highlight.js.git"
   },
+  "type": "commonjs",
   "main": "./lib/index.js",
   "types": "./types/index.d.ts",
   "scripts": {

src/core.d.ts

@@ -1,6 +1,3 @@
-// commented out
-// <reference path="../types/index.d.ts" />
-
 import hljs from "highlight.js";
 export default hljs;
 

tools/build.js

@@ -71,6 +71,7 @@ const dir = {};
 commander
   .usage('[options] [<language>...]')
   .option('-n, --no-minify', 'Disable minification')
+  .option('-ne, --no-esm', 'Disable building ESM')
   .option('-t, --target <name>',
     'Build for target ' +
     '[all, browser, cdn, node]',
@@ -86,7 +87,7 @@ async function doTarget(target, buildDir) {
   const build = require(`./build_${target}`);
   process.env.BUILD_DIR = buildDir;
   await clean(buildDir);
-  await build.build({ languages: commander.args, minify: commander.opts().minify });
+  await build.build({ languages: commander.args, minify: commander.opts().minify, esm: commander.opts().esm });
 }
 
 async function doBuild() {

tools/build_node.js

@@ -6,86 +6,193 @@ const { filter } = require("./lib/dependencies");
 const { rollupWrite } = require("./lib/bundling.js");
 const log = (...args) => console.log(...args);
 
-async function buildNodeIndex(languages) {
+const safeImportName = (s) => {
+  s = s.replace(/-/g, "_");
+  if (/^\d/.test(s)) s = `L_${s}`;
+  return s;
+};
+
+async function buildESMIndex(name, languages) {
+  const header = `import hljs from './core.js';`;
+  const footer = "export default hljs;";
+
+
+  const registration = languages.map((lang) => {
+    const importName = safeImportName(lang.name);
+    return `import ${importName} from './languages/${lang.name}.js';\n` +
+      `hljs.registerLanguage('${lang.name}', ${importName});`;
+  });
+
+  const index = `${header}\n\n${registration.join("\n")}\n\n${footer}`;
+  await fs.writeFile(`${process.env.BUILD_DIR}/es/${name}.js`, index);
+}
+
+async function buildCJSIndex(name, languages) {
   const header = "var hljs = require('./core');";
   const footer = "module.exports = hljs;";
 
   const registration = languages.map((lang) => {
-    let require = `require('./languages/${lang.name}')`;
-    if (lang.loader) {
-      require = require += `.${lang.loader}`;
-    }
+    const require = `require('./languages/${lang.name}')`;
     return `hljs.registerLanguage('${lang.name}', ${require});`;
   });
 
-  // legacy
-  await fs.writeFile(`${process.env.BUILD_DIR}/lib/highlight.js`,
-    "// This file has been deprecated in favor of core.js\n" +
-    "var hljs = require('./core');\n"
-  );
-
   const index = `${header}\n\n${registration.join("\n")}\n\n${footer}`;
-  await fs.writeFile(`${process.env.BUILD_DIR}/lib/index.js`, index);
+  await fs.writeFile(`${process.env.BUILD_DIR}/lib/${name}.js`, index);
 }
 
-async function buildNodeLanguage(language) {
+async function buildNodeLanguage(language, options) {
+  const EMIT = `function emitWarning() {
+    if (!emitWarning.warned) {
+      emitWarning.warned = true;
+      process.emitWarning(
+        'Using file extension in specifier is deprecated, use "highlight.js/lib/languages/%%%%" instead of "highlight.js/lib/languages/%%%%.js"',
+        'DeprecationWarning'
+      );
+    }
+  }
+  emitWarning();`;
+  const CJS_STUB = `${EMIT}
+    module.exports = require('./%%%%.js');`;
+  const ES_STUB = `${EMIT}
+    import lang from './%%%%.js';
+    export default lang;`;
   const input = { ...config.rollup.node.input, input: language.path };
   const output = { ...config.rollup.node.output, file: `${process.env.BUILD_DIR}/lib/languages/${language.name}.js` };
   await rollupWrite(input, output);
+  await fs.writeFile(`${process.env.BUILD_DIR}/lib/languages/${language.name}.js.js`,
+    CJS_STUB.replace(/%%%%/g, language.name));
+  if (options.esm) {
+    await fs.writeFile(`${process.env.BUILD_DIR}/es/languages/${language.name}.js.js`,
+      ES_STUB.replace(/%%%%/g, language.name));
+    await rollupWrite(input, {...output,
+      format: "es",
+      file: output.file.replace("/lib/", "/es/")
+    });
+  }
 }
 
-async function buildNodeHighlightJS() {
+const EXCLUDE = ["join"];
+
+async function buildESMUtils() {
+  const input = { ...config.rollup.node.input, input: `src/lib/regex.js` };
+  input.plugins = [...input.plugins, {
+    transform: (code) => {
+      EXCLUDE.forEach((fn) => {
+        code = code.replace(`export function ${fn}(`, `function ${fn}(`);
+      });
+      return code;
+    }
+  }];
+  await rollupWrite(input, {
+    ...config.rollup.node.output,
+    format: "es",
+    file: `${process.env.BUILD_DIR}/es/utils/regex.js`
+  });
+}
+
+async function buildNodeHighlightJS(options) {
   const input = { ...config.rollup.node.input, input: `src/highlight.js` };
   const output = { ...config.rollup.node.output, file: `${process.env.BUILD_DIR}/lib/core.js` };
   await rollupWrite(input, output);
+  if (options.esm) {
+    await rollupWrite(input, {
+      ...output,
+      format: "es",
+      file: `${process.env.BUILD_DIR}/es/core.js`
+    });
+  }
 }
 
-async function buildPackageJSON() {
+function dual(file) {
+  return {
+    require: file,
+    import: file.replace("/lib/", "/es/")
+  };
+}
+
+async function buildPackageJSON(options) {
   const packageJson = require("../package");
+
+  const exports = {
+    ".": dual("./lib/index.js"),
+    "./package.json": "./package.json",
+    "./lib/common": dual("./lib/common.js"),
+    "./lib/core": dual("./lib/core.js"),
+    "./lib/languages/*": dual("./lib/languages/*.js"),
+  };
+  if (options.esm) packageJson.exports = exports;
+
   await fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(packageJson, null, 2));
 }
 
-async function buildLanguages(languages) {
+async function buildLanguages(languages, options) {
   log("Writing languages.");
   await Promise.all(
     languages.map(async(lang) => {
-      await buildNodeLanguage(lang);
+      await buildNodeLanguage(lang, options);
       process.stdout.write(".");
     })
   );
   log("");
 }
 
+const CORE_FILES = [
+  "LICENSE",
+  "README.md",
+  "VERSION_10_UPGRADE.md",
+  "VERSION_11_UPGRADE.md",
+  "SUPPORTED_LANGUAGES.md",
+  "SECURITY.md",
+  "CHANGES.md",
+  "types/index.d.ts"
+];
+
 async function buildNode(options) {
   mkdir("lib/languages");
   mkdir("scss");
   mkdir("styles");
   mkdir("types");
 
-  install("./LICENSE", "LICENSE");
-  install("./README.md", "README.md");
-  install("./types/index.d.ts", "types/index.d.ts");
-  install("./src/core.d.ts","lib/core.d.ts");
+
+  CORE_FILES.forEach(file => {
+    install(`./${file}`, file);
+  });
+  install("./src/core.d.ts", "lib/core.d.ts");
+  install("./src/core.d.ts", "lib/common.d.ts");
+
+  if (options.esm) {
+    mkdir("es/languages");
+    install("./src/core.d.ts", "es/core.d.ts");
+    install("./src/core.d.ts", "es/common.d.ts");
+  }
 
   log("Writing styles.");
   const styles = await fs.readdir("./src/styles/");
   styles.forEach((file) => {
     install(`./src/styles/${file}`, `styles/${file}`);
     install(`./src/styles/${file}`, `scss/${file.replace(".css", ".scss")}`);
   });
-  log("Writing package.json.");
-  await buildPackageJSON();
 
   let languages = await getLanguages();
   // filter languages for inclusion in the highlight.js bundle
   languages = filter(languages, options.languages);
+  const common = languages.filter(l => l.categories.includes("common"));
 
-  await buildNodeIndex(languages);
-  await buildLanguages(languages);
+  log("Writing package.json.");
+  await buildPackageJSON(options);
+
+  if (options.esm) {
+    await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`);
+    await buildESMIndex("index", languages);
+    await buildESMIndex("common", common);
+    await buildESMUtils();
+  }
+  await buildCJSIndex("index", languages);
+  await buildCJSIndex("common", common);
+  await buildLanguages(languages, options);
 
   log("Writing highlight.js");
-  await buildNodeHighlightJS();
-
+  await buildNodeHighlightJS(options);
 }
 
 module.exports.build = buildNode;