refactor: replace module-deps-sortable on own implementation to control how parse files

BREAKING CHANGE:
all Extensions whould contains '.' so that mean if you have just 'ts' then need to convert to '.ts'

Author: anthony-redFox

__tests__/lib/input/dependency.js

@@ -23,13 +23,13 @@ test('dependency', async function () {
   });
   {
     const dependencies = await dependency([paths['index.js']], {
-      parseExtension: ['js']
+      parseExtension: ['.js']
     });
     expect(dependencies.length).toEqual(1);
   }
   {
     const dependencies = await dependency([paths['requires.js']], {
-      parseExtension: ['js']
+      parseExtension: ['.js']
     });
     expect(dependencies.length).toEqual(2);
   }

docs/NODE_API.md

@@ -126,14 +126,9 @@ Formats documentation as HTML.
 
 ```javascript
 var documentation = require('documentation');
-var streamArray = require('stream-array');
-var vfs = require('vinyl-fs');
 
 documentation.build(['index.js'])
-  .then(documentation.formats.html)
-  .then(output => {
-    streamArray(output).pipe(vfs.dest('./output-directory'));
-  });
+  .then(documentation.formats.html);
 ```
 
 Returns **[Promise][21]<[Array][17]<[Object][19]>>** Promise with results

package-lock.json

@@ -16,7 +16,6 @@
     "@babel/types": "^7.14.1",
     "chalk": "^4.1.2",
     "chokidar": "^3.4.0",
-    "concat-stream": "^2.0.0",
     "diff": "^5.0.0",
     "doctrine-temporary-fork": "2.1.0",
     "git-url-parse": "^11.1.2",
@@ -30,7 +29,6 @@
     "mdast-util-find-and-replace": "^2.1.0",
     "mdast-util-inject": "^1.1.0",
     "micromark-util-character": "^1.1.0",
-    "module-deps-sortable": "^5.0.3",
     "parse-filepath": "^1.0.2",
     "pify": "^5.0.0",
     "read-pkg-up": "^9.0.0",
@@ -39,8 +37,9 @@
     "remark-html": "^15.0.0",
     "remark-reference-links": "^6.0.0",
     "remark-toc": "^8.0.0",
+    "detective": "^5.2.0",
+    "konan": "^2.1.1",
     "resolve": "^1.8.1",
-    "stream-array": "^1.1.2",
     "strip-json-comments": "^4.0.0",
     "unist-builder": "^3.0.0",
     "unist-util-visit": "^4.1.0",
@@ -59,6 +58,7 @@
     "documentation-schema": "0.0.1",
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",
+    "concat-stream": "^2.0.0",
     "fs-extra": "^10.0.0",
     "husky": "^4.3.8",
     "jest": "^27.1.0",

package.json

@@ -2,7 +2,7 @@ const defaultConfig = {
   // package.json ignored and don't get project infromation
   'no-package': false,
   // Extenstions which by dafault are parse
-  parseExtension: ['mjs', 'js', 'jsx', 'es5', 'es6', 'vue', 'ts', 'tsx']
+  parseExtension: ['.mjs', '.js', '.jsx', '.es5', '.es6', '.vue', '.ts', '.tsx']
 };
 
 function normalaze(config, global) {

src/config.js

@@ -1,11 +1,9 @@
-import mdeps from 'module-deps-sortable';
-import path from 'path';
-import concat from 'concat-stream';
+import mdeps from './moduleDeps.js';
 import internalOnly from '../module_filters.js';
 import smartGlob from '../smart_glob.js';
 
 /**
- * Returns a readable stream of dependencies, given an array of entry
+ * Returns a array of dependencies, given an array of entry
  * points and an object of options to provide to module-deps.
  *
  * This stream requires filesystem access, and thus isn't suitable
@@ -15,55 +13,19 @@ import smartGlob from '../smart_glob.js';
  * @param config optional options passed
  * @returns results
  */
-export default function dependencyStream(indexes, config) {
-  const md = mdeps({
+export default async function dependencyStream(
+  indexes,
+  { parseExtension = [], requireExtension = [] }
+) {
+  const md = await mdeps(smartGlob(indexes, parseExtension), {
     /**
      * Determine whether a module should be included in documentation
      * @param {string} id path to a module
      * @returns {boolean} true if the module should be included.
      */
     filter: id => internalOnly(id),
-    extensions: []
-      .concat(config.requireExtension || [])
-      .map(ext => '.' + ext.replace(/^\./, ''))
-      .concat(['.mjs', '.js', '.json', '.es6', '.jsx']),
-    resolve:
-      config.resolve === 'node' &&
-      ((id, opts, cb) => {
-        const r = require('resolve');
-        opts.basedir = path.dirname(opts.filename);
-        r(id, opts, cb);
-      })
+    extensions: [...parseExtension, ...requireExtension]
   });
-  smartGlob(indexes, config.parseExtension).forEach(index => {
-    md.write(path.resolve(index));
-  });
-  md.end();
 
-  return new Promise((resolve, reject) => {
-    md.once('error', reject);
-    md.pipe(
-      concat(function (inputs) {
-        resolve(
-          inputs
-            .filter(
-              input =>
-                // At this point, we may have allowed a JSON file to be caught by
-                // module-deps, or anything else allowed by requireExtension.
-                // otherwise module-deps would complain about
-                // it not being found. But Babel can't parse JSON, so we filter non-JavaScript
-                // files away.
-                config.parseExtension.indexOf(
-                  path.extname(input.file).replace(/^\./, '')
-                ) > -1
-            )
-            .map(input => {
-              // remove source file, since it's transformed anyway
-              delete input.source;
-              return input;
-            })
-        );
-      })
-    );
-  });
+  return md;
 }

src/input/dependency.js

@@ -0,0 +1,114 @@
+import path from 'path';
+import util from 'util';
+import { readFile } from 'fs/promises';
+import r from 'resolve';
+import detective from 'detective';
+import konan from 'konan';
+
+// const parseExst = ['.js', '.mjs', '.jsx', '.vue', '.ts', '.tsx'];
+const resolveExst = ['.json', '.css', '.less', '.sass'];
+const resolve = util.promisify(r);
+
+class Deps {
+  constructor(opts = {}) {
+    this.fileCache = opts.fileCache || {};
+    this.visited = {};
+    this.res = [];
+
+    this.options = { ...opts };
+  }
+
+  async flush(input) {
+    const promises = input.map(file => {
+      const dir = path.dirname(file);
+      return this.walk(file, {
+        basedir: dir,
+        filename: 'root'
+      });
+    });
+    await Promise.all(promises);
+
+    return this.res;
+  }
+
+  async readFile(file) {
+    if (this.fileCache[file]) {
+      return this.fileCache[file];
+    }
+    return readFile(file, {
+      encoding: 'utf8'
+    });
+  }
+
+  async walk(id, parent) {
+    const extensions = this.options.extensions;
+    const sortKey = parent.sortKey || '';
+    let file = null;
+
+    try {
+      file = await resolve(id, { ...parent, extensions });
+    } catch (err) {
+      if (err.code === 'MODULE_NOT_FOUND') {
+        console.warn(`module not found: "${id}" from file ${parent.filename}`);
+        return;
+      }
+      throw err;
+    }
+
+    if (this.visited[file] || resolveExst.includes(path.extname(file))) {
+      return file;
+    }
+    this.visited[file] = true;
+
+    const source = await this.readFile(file);
+    const depsArray = this.parseDeps(file, source);
+    if (!depsArray) {
+      return file;
+    }
+
+    const deps = {};
+    const promises = depsArray.map(async (id, i) => {
+      const filter = this.options.filter;
+      if (filter && !filter(id)) {
+        deps[id] = false;
+        return;
+      }
+      const number = i.toString().padStart(8, '0');
+      deps[id] = await this.walk(id, {
+        filename: file,
+        basedir: path.dirname(file),
+        sortKey: sortKey + '!' + file + ':' + number
+      });
+    });
+
+    await Promise.all(promises);
+
+    this.res.push({
+      id: file,
+      source,
+      deps,
+      file,
+      sortKey: sortKey + '!' + file
+    });
+    return file;
+  }
+
+  parseDeps(file, src) {
+    try {
+      try {
+        return konan(src).strings;
+      } catch (ex) {
+        // konan does not support Vue (component) file, try to parse using detective (as a fallback)
+        return detective(src);
+      }
+    } catch (ex) {
+      console.error(`Parsing file ${file}: ${ex}`);
+      return;
+    }
+  }
+}
+
+export default async function (input = [], opts = {}) {
+  const dep = new Deps(opts);
+  return dep.flush(Array.from(new Set(input)));
+}

src/input/moduleDeps.js

@@ -12,14 +12,9 @@ import mergeConfig from '../merge_config.js';
  * @public
  * @example
  * var documentation = require('documentation');
- * var streamArray = require('stream-array');
- * var vfs = require('vinyl-fs');
  *
  * documentation.build(['index.js'])
- *   .then(documentation.formats.html)
- *   .then(output => {
- *     streamArray(output).pipe(vfs.dest('./output-directory'));
- *   });
+ *   .then(documentation.formats.html);
  */
 export default async function html(comments, localConfig = {}) {
   const config = await mergeConfig(localConfig);