Split extension logic into requireExtension and parseExtension

This changes logic quite a bit: no more filter_js and better
thought-through settings that differentiate between require extensions,
which are necessary to keep module-deps from crashing, and parse
extensions, which add new filetypes to the range that we think are
parseable.

I don't think this is done yet, unfortunately, because smart_glob
uses parse extensions as inputs, and that goes kind of against
how polyglot/shallow mode is probably expected to work: right now
you'd have to add --parseExtension cpp, for instance, for C++ parsing.

Author: tmcw

index.js

@@ -26,6 +26,8 @@ var fs = require('fs'),
   markdownAST = require('./lib/output/markdown_ast'),
   loadConfig = require('./lib/load_config');
 
+var parseExtensions = ['js', 'jsx', 'es5', 'es6'];
+
 /**
  * Build a pipeline of comment handlers.
  * @param {...Function|null} args - Pipeline elements. Each is a function that accepts
@@ -61,6 +63,8 @@ function expandInputs(indexes, options, callback) {
   } else {
     inputFn = dependency;
   }
+  options.parseExtensions = parseExtensions
+    .concat(options.parseExtension || []);
   inputFn(indexes, options, callback);
 }
 

lib/commands/shared_options.js

@@ -24,10 +24,22 @@ module.exports.sharedInputOptions = {
       'modules will be whitelisted and included in the generated documentation.',
     default: null
   },
-  'extension': {
-    describe: 'only input source files matching this extension will be parsed, ' +
-      'this option can be used multiple times.',
-    alias: 'e'
+  'requireExtension': {
+    describe: 'additional extensions to include in require() and import\'s search algorithm.' +
+      'For instance, adding .es5 would allow require("adder") to find "adder.es5"',
+    coerce: function (value) {
+      // Ensure that the value is an array
+      return [].concat(value);
+    },
+    alias: 're'
+  },
+  'parseExtension': {
+    describe: 'additional extensions to parse as source code.',
+    coerce: function (value) {
+      // Ensure that the value is an array
+      return [].concat(value);
+    },
+    alias: 'pe'
   },
   'polyglot': {
     type: 'boolean',

lib/filter_js.js

@@ -4,7 +4,6 @@ var mdeps = require('module-deps-sortable');
 var fs = require('fs');
 var path = require('path');
 var babelify = require('babelify');
-var filterJS = require('../filter_js');
 var concat = require('concat-stream');
 var moduleFilters = require('../../lib/module_filters');
 var smartGlob = require('../smart_glob.js');
@@ -22,7 +21,6 @@ var smartGlob = require('../smart_glob.js');
  * @returns {undefined} calls callback
  */
 function dependencyStream(indexes, options, callback) {
-  var filterer = filterJS(options.extension, options.polyglot);
   var md = mdeps({
     /**
      * Determine whether a module should be included in documentation
@@ -32,14 +30,11 @@ function dependencyStream(indexes, options, callback) {
     filter: function (id) {
       return !!options.external || moduleFilters.internalOnly(id);
     },
-    extensions: [].concat(options.extension || [])
-      // We don't document JSON files, but we do include them in this list,
-      // because browserify & node both support unnamed requires to .json
-      // files: `require('foo')` can require `foo.json` automatically.
-      .concat(['js', 'es6', 'jsx', 'json'])
+    extensions: [].concat(options.requireExtension || [])
       .map(function (ext) {
-        return '.' + ext;
-      }),
+        return '.' + ext.replace(/^\./, '');
+      })
+      .concat(['.js', '.json', '.es6', '.jsx']),
     transform: [babelify.configure({
       sourceMap: false,
       compact: false,
@@ -57,7 +52,7 @@ function dependencyStream(indexes, options, callback) {
     })],
     postFilter: moduleFilters.externals(indexes, options)
   });
-  smartGlob(indexes, options.extension).forEach(function (index) {
+  smartGlob(indexes, options.parseExtensions).forEach(function (index) {
     md.write(path.resolve(index));
   });
   md.end();
@@ -66,7 +61,16 @@ function dependencyStream(indexes, options, callback) {
   });
   md.pipe(concat(function (inputs) {
     callback(null, inputs
-      .filter(filterer)
+      .filter(function (input) {
+        // At this point, we may have allowed a JSON file to be caught by
+        // module-deps, or anything else allowed by requireExtension.
+        // otherwise module-deps would complain about
+        // it not being found. But Babel can't parse JSON, so we filter non-JavaScript
+        // files away.
+        return options.parseExtensions.indexOf(
+          path.extname(input.file).replace(/^\./, '')
+        ) > -1;
+      })
       .map(function (input) {
         // un-transform babelify transformed source
         input.source = fs.readFileSync(input.file, 'utf8');

lib/input/dependency.js

@@ -28,8 +28,8 @@ module.exports = function (indexes, options, callback) {
     } else if (typeof index === 'object') {
       objects.push(index);
     } else {
-      throw new Error('indexes should be either strings or objects');
+      throw new Error('Indexes should be either strings or objects');
     }
   });
-  return callback(null, objects.concat(smartGlob(strings, options.extension)));
+  return callback(null, objects.concat(smartGlob(strings, options.parseExtensions)));
 };

lib/input/shallow.js

@@ -35,7 +35,7 @@ function convertPathToPosix(filepath) {
  */
 function processPath(options) {
   var cwd = (options && options.cwd) || process.cwd();
-  var extensions = (options && options.extensions) || ['.js'];
+  var extensions = (options && options.requireExtension) || ['.js'];
 
   extensions = extensions.map(function (ext) {
     return ext.replace(/^\./, '');

lib/smart_glob.js

@@ -62,7 +62,7 @@
     "standard-changelog": "0.0.1",
     "cz-conventional-changelog": "1.2.0",
     "documentation-schema": "0.0.1",
-    "eslint": "^3.1.0",
+    "eslint": "^3.12.2",
     "fs-extra": "^1.0.0",
     "json-schema": "0.2.3",
     "mock-fs": "^3.5.0",

package.json

@@ -141,7 +141,7 @@ test('external modules option', function (t) {
 
 test('extension option', function (t) {
   documentation(['build fixture/extension/index.otherextension ' +
-    '--extension=otherextension'], function (err, data) {
+    '--requireExtension=otherextension --parseExtension=otherextension'], function (err, data) {
     t.ifError(err);
     t.equal(data.length, 1, 'includes a file with an arbitrary extension');
     t.end();

test/bin.js

@@ -1 +1 @@
-{}
+{"foo":"bar"}

test/fixture/require-json.json

@@ -27,11 +27,11 @@ test('polyglot', function (t) {
         description: remark().parse('color'),
         type: { name: 'number', type: 'NameExpression' } } ],
     tags: [ { description: null, lineNumber: 2, name: 'hexToUInt32Color', title: 'name' },
-    { description: null, lineNumber: 3, name: 'hex', title: 'param', type: {
-      name: 'string', type: 'NameExpression'
-    } },
-    { description: 'color', lineNumber: 4, title: 'returns', type: {
-      name: 'number', type: 'NameExpression'
-    } } ] } ], 'polyglot parser');
+      { description: null, lineNumber: 3, name: 'hex', title: 'param', type: {
+        name: 'string', type: 'NameExpression'
+      } },
+      { description: 'color', lineNumber: 4, title: 'returns', type: {
+        name: 'number', type: 'NameExpression'
+      } } ] } ], 'polyglot parser');
   t.end();
 });